diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common/fb')
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gk20a.c | 123 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gk20a.h | 33 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gm20b.c | 229 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gm20b.h | 47 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gp106.c | 51 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gp106.h | 28 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gp10b.c | 38 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gp10b.h | 32 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gv100.c | 257 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gv100.h | 36 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gv11b.c | 1516 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gv11b.h | 82 |
12 files changed, 2472 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gk20a.c b/drivers/gpu/nvgpu/common/fb/fb_gk20a.c new file mode 100644 index 00000000..d27ac9d0 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gk20a.c | |||
@@ -0,0 +1,123 @@ | |||
1 | /* | ||
2 | * GK20A memory interface | ||
3 | * | ||
4 | * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <trace/events/gk20a.h> | ||
26 | |||
27 | #include "gk20a/gk20a.h" | ||
28 | |||
29 | #include "fb_gk20a.h" | ||
30 | |||
31 | #include <nvgpu/timers.h> | ||
32 | |||
33 | #include <nvgpu/hw/gk20a/hw_mc_gk20a.h> | ||
34 | #include <nvgpu/hw/gk20a/hw_fb_gk20a.h> | ||
35 | |||
36 | void fb_gk20a_reset(struct gk20a *g) | ||
37 | { | ||
38 | u32 val; | ||
39 | |||
40 | nvgpu_log_info(g, "reset gk20a fb"); | ||
41 | |||
42 | g->ops.mc.reset(g, mc_enable_pfb_enabled_f() | | ||
43 | mc_enable_l2_enabled_f() | | ||
44 | mc_enable_xbar_enabled_f() | | ||
45 | mc_enable_hub_enabled_f()); | ||
46 | |||
47 | val = gk20a_readl(g, mc_elpg_enable_r()); | ||
48 | val |= mc_elpg_enable_xbar_enabled_f() | ||
49 | | mc_elpg_enable_pfb_enabled_f() | ||
50 | | mc_elpg_enable_hub_enabled_f(); | ||
51 | gk20a_writel(g, mc_elpg_enable_r(), val); | ||
52 | } | ||
53 | |||
54 | void gk20a_fb_init_hw(struct gk20a *g) | ||
55 | { | ||
56 | u32 addr = nvgpu_mem_get_addr(g, &g->mm.sysmem_flush) >> 8; | ||
57 | |||
58 | gk20a_writel(g, fb_niso_flush_sysmem_addr_r(), addr); | ||
59 | } | ||
60 | |||
61 | void gk20a_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb) | ||
62 | { | ||
63 | struct nvgpu_timeout timeout; | ||
64 | u32 addr_lo; | ||
65 | u32 data; | ||
66 | |||
67 | nvgpu_log_fn(g, " "); | ||
68 | |||
69 | /* pagetables are considered sw states which are preserved after | ||
70 | prepare_poweroff. When gk20a deinit releases those pagetables, | ||
71 | common code in vm unmap path calls tlb invalidate that touches | ||
72 | hw. Use the power_on flag to skip tlb invalidation when gpu | ||
73 | power is turned off */ | ||
74 | |||
75 | if (!g->power_on) | ||
76 | return; | ||
77 | |||
78 | addr_lo = u64_lo32(nvgpu_mem_get_addr(g, pdb) >> 12); | ||
79 | |||
80 | nvgpu_mutex_acquire(&g->mm.tlb_lock); | ||
81 | |||
82 | trace_gk20a_mm_tlb_invalidate(g->name); | ||
83 | |||
84 | nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER); | ||
85 | |||
86 | do { | ||
87 | data = gk20a_readl(g, fb_mmu_ctrl_r()); | ||
88 | if (fb_mmu_ctrl_pri_fifo_space_v(data) != 0) | ||
89 | break; | ||
90 | nvgpu_udelay(2); | ||
91 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
92 | "wait mmu fifo space")); | ||
93 | |||
94 | if (nvgpu_timeout_peek_expired(&timeout)) | ||
95 | goto out; | ||
96 | |||
97 | nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER); | ||
98 | |||
99 | gk20a_writel(g, fb_mmu_invalidate_pdb_r(), | ||
100 | fb_mmu_invalidate_pdb_addr_f(addr_lo) | | ||
101 | nvgpu_aperture_mask(g, pdb, | ||
102 | fb_mmu_invalidate_pdb_aperture_sys_mem_f(), | ||
103 | fb_mmu_invalidate_pdb_aperture_sys_mem_f(), | ||
104 | fb_mmu_invalidate_pdb_aperture_vid_mem_f())); | ||
105 | |||
106 | gk20a_writel(g, fb_mmu_invalidate_r(), | ||
107 | fb_mmu_invalidate_all_va_true_f() | | ||
108 | fb_mmu_invalidate_trigger_true_f()); | ||
109 | |||
110 | do { | ||
111 | data = gk20a_readl(g, fb_mmu_ctrl_r()); | ||
112 | if (fb_mmu_ctrl_pri_fifo_empty_v(data) != | ||
113 | fb_mmu_ctrl_pri_fifo_empty_false_f()) | ||
114 | break; | ||
115 | nvgpu_udelay(2); | ||
116 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
117 | "wait mmu invalidate")); | ||
118 | |||
119 | trace_gk20a_mm_tlb_invalidate_done(g->name); | ||
120 | |||
121 | out: | ||
122 | nvgpu_mutex_release(&g->mm.tlb_lock); | ||
123 | } | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gk20a.h b/drivers/gpu/nvgpu/common/fb/fb_gk20a.h new file mode 100644 index 00000000..072c9027 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gk20a.h | |||
@@ -0,0 +1,33 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifndef FB_GK20A_H | ||
24 | #define FB_GK20A_H | ||
25 | |||
26 | struct gk20a; | ||
27 | struct nvgpu_mem; | ||
28 | |||
29 | void fb_gk20a_reset(struct gk20a *g); | ||
30 | void gk20a_fb_init_hw(struct gk20a *g); | ||
31 | void gk20a_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb); | ||
32 | |||
33 | #endif | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gm20b.c b/drivers/gpu/nvgpu/common/fb/fb_gm20b.c new file mode 100644 index 00000000..bd093b31 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gm20b.c | |||
@@ -0,0 +1,229 @@ | |||
1 | /* | ||
2 | * GM20B GPC MMU | ||
3 | * | ||
4 | * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <nvgpu/sizes.h> | ||
26 | |||
27 | #include "gk20a/gk20a.h" | ||
28 | |||
29 | #include "fb_gk20a.h" | ||
30 | #include "fb_gm20b.h" | ||
31 | |||
32 | #include <nvgpu/hw/gm20b/hw_fb_gm20b.h> | ||
33 | #include <nvgpu/hw/gm20b/hw_top_gm20b.h> | ||
34 | #include <nvgpu/hw/gm20b/hw_gmmu_gm20b.h> | ||
35 | #include <nvgpu/hw/gm20b/hw_gr_gm20b.h> | ||
36 | |||
37 | #define VPR_INFO_FETCH_WAIT (5) | ||
38 | #define WPR_INFO_ADDR_ALIGNMENT 0x0000000c | ||
39 | |||
40 | void fb_gm20b_init_fs_state(struct gk20a *g) | ||
41 | { | ||
42 | nvgpu_log_info(g, "initialize gm20b fb"); | ||
43 | |||
44 | gk20a_writel(g, fb_fbhub_num_active_ltcs_r(), | ||
45 | g->ltc_count); | ||
46 | |||
47 | if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) { | ||
48 | /* Bypass MMU check for non-secure boot. For | ||
49 | * secure-boot,this register write has no-effect */ | ||
50 | gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffffU); | ||
51 | } | ||
52 | } | ||
53 | |||
54 | void gm20b_fb_set_mmu_page_size(struct gk20a *g) | ||
55 | { | ||
56 | /* set large page size in fb */ | ||
57 | u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r()); | ||
58 | fb_mmu_ctrl |= fb_mmu_ctrl_use_pdb_big_page_size_true_f(); | ||
59 | gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl); | ||
60 | } | ||
61 | |||
62 | bool gm20b_fb_set_use_full_comp_tag_line(struct gk20a *g) | ||
63 | { | ||
64 | /* set large page size in fb */ | ||
65 | u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r()); | ||
66 | fb_mmu_ctrl |= fb_mmu_ctrl_use_full_comp_tag_line_true_f(); | ||
67 | gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl); | ||
68 | |||
69 | return true; | ||
70 | } | ||
71 | |||
72 | u32 gm20b_fb_mmu_ctrl(struct gk20a *g) | ||
73 | { | ||
74 | return gk20a_readl(g, fb_mmu_ctrl_r()); | ||
75 | } | ||
76 | |||
77 | u32 gm20b_fb_mmu_debug_ctrl(struct gk20a *g) | ||
78 | { | ||
79 | return gk20a_readl(g, fb_mmu_debug_ctrl_r()); | ||
80 | } | ||
81 | |||
82 | u32 gm20b_fb_mmu_debug_wr(struct gk20a *g) | ||
83 | { | ||
84 | return gk20a_readl(g, fb_mmu_debug_wr_r()); | ||
85 | } | ||
86 | |||
87 | u32 gm20b_fb_mmu_debug_rd(struct gk20a *g) | ||
88 | { | ||
89 | return gk20a_readl(g, fb_mmu_debug_rd_r()); | ||
90 | } | ||
91 | |||
92 | unsigned int gm20b_fb_compression_page_size(struct gk20a *g) | ||
93 | { | ||
94 | return SZ_128K; | ||
95 | } | ||
96 | |||
97 | unsigned int gm20b_fb_compressible_page_size(struct gk20a *g) | ||
98 | { | ||
99 | return SZ_64K; | ||
100 | } | ||
101 | |||
102 | u32 gm20b_fb_compression_align_mask(struct gk20a *g) | ||
103 | { | ||
104 | return SZ_64K - 1; | ||
105 | } | ||
106 | |||
107 | void gm20b_fb_dump_vpr_wpr_info(struct gk20a *g) | ||
108 | { | ||
109 | u32 val; | ||
110 | |||
111 | /* print vpr and wpr info */ | ||
112 | val = gk20a_readl(g, fb_mmu_vpr_info_r()); | ||
113 | val &= ~0x3; | ||
114 | val |= fb_mmu_vpr_info_index_addr_lo_v(); | ||
115 | gk20a_writel(g, fb_mmu_vpr_info_r(), val); | ||
116 | nvgpu_err(g, "VPR: %08x %08x %08x %08x", | ||
117 | gk20a_readl(g, fb_mmu_vpr_info_r()), | ||
118 | gk20a_readl(g, fb_mmu_vpr_info_r()), | ||
119 | gk20a_readl(g, fb_mmu_vpr_info_r()), | ||
120 | gk20a_readl(g, fb_mmu_vpr_info_r())); | ||
121 | |||
122 | val = gk20a_readl(g, fb_mmu_wpr_info_r()); | ||
123 | val &= ~0xf; | ||
124 | val |= (fb_mmu_wpr_info_index_allow_read_v()); | ||
125 | gk20a_writel(g, fb_mmu_wpr_info_r(), val); | ||
126 | nvgpu_err(g, "WPR: %08x %08x %08x %08x %08x %08x", | ||
127 | gk20a_readl(g, fb_mmu_wpr_info_r()), | ||
128 | gk20a_readl(g, fb_mmu_wpr_info_r()), | ||
129 | gk20a_readl(g, fb_mmu_wpr_info_r()), | ||
130 | gk20a_readl(g, fb_mmu_wpr_info_r()), | ||
131 | gk20a_readl(g, fb_mmu_wpr_info_r()), | ||
132 | gk20a_readl(g, fb_mmu_wpr_info_r())); | ||
133 | |||
134 | } | ||
135 | |||
136 | static int gm20b_fb_vpr_info_fetch_wait(struct gk20a *g, | ||
137 | unsigned int msec) | ||
138 | { | ||
139 | struct nvgpu_timeout timeout; | ||
140 | |||
141 | nvgpu_timeout_init(g, &timeout, msec, NVGPU_TIMER_CPU_TIMER); | ||
142 | |||
143 | do { | ||
144 | u32 val; | ||
145 | |||
146 | val = gk20a_readl(g, fb_mmu_vpr_info_r()); | ||
147 | if (fb_mmu_vpr_info_fetch_v(val) == | ||
148 | fb_mmu_vpr_info_fetch_false_v()) | ||
149 | return 0; | ||
150 | |||
151 | } while (!nvgpu_timeout_expired(&timeout)); | ||
152 | |||
153 | return -ETIMEDOUT; | ||
154 | } | ||
155 | |||
156 | int gm20b_fb_vpr_info_fetch(struct gk20a *g) | ||
157 | { | ||
158 | if (gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT)) { | ||
159 | return -ETIMEDOUT; | ||
160 | } | ||
161 | |||
162 | gk20a_writel(g, fb_mmu_vpr_info_r(), | ||
163 | fb_mmu_vpr_info_fetch_true_v()); | ||
164 | |||
165 | return gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT); | ||
166 | } | ||
167 | |||
168 | void gm20b_fb_read_wpr_info(struct gk20a *g, struct wpr_carveout_info *inf) | ||
169 | { | ||
170 | u32 val = 0; | ||
171 | u64 wpr_start = 0; | ||
172 | u64 wpr_end = 0; | ||
173 | |||
174 | val = gk20a_readl(g, fb_mmu_wpr_info_r()); | ||
175 | val &= ~0xF; | ||
176 | val |= fb_mmu_wpr_info_index_wpr1_addr_lo_v(); | ||
177 | gk20a_writel(g, fb_mmu_wpr_info_r(), val); | ||
178 | |||
179 | val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4; | ||
180 | wpr_start = hi32_lo32_to_u64( | ||
181 | (val >> (32 - WPR_INFO_ADDR_ALIGNMENT)), | ||
182 | (val << WPR_INFO_ADDR_ALIGNMENT)); | ||
183 | |||
184 | val = gk20a_readl(g, fb_mmu_wpr_info_r()); | ||
185 | val &= ~0xF; | ||
186 | val |= fb_mmu_wpr_info_index_wpr1_addr_hi_v(); | ||
187 | gk20a_writel(g, fb_mmu_wpr_info_r(), val); | ||
188 | |||
189 | val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4; | ||
190 | wpr_end = hi32_lo32_to_u64( | ||
191 | (val >> (32 - WPR_INFO_ADDR_ALIGNMENT)), | ||
192 | (val << WPR_INFO_ADDR_ALIGNMENT)); | ||
193 | |||
194 | inf->wpr_base = wpr_start; | ||
195 | inf->nonwpr_base = 0; | ||
196 | inf->size = (wpr_end - wpr_start); | ||
197 | } | ||
198 | |||
199 | bool gm20b_fb_debug_mode_enabled(struct gk20a *g) | ||
200 | { | ||
201 | u32 debug_ctrl = gk20a_readl(g, gr_gpcs_pri_mmu_debug_ctrl_r()); | ||
202 | return gr_gpcs_pri_mmu_debug_ctrl_debug_v(debug_ctrl) == | ||
203 | gr_gpcs_pri_mmu_debug_ctrl_debug_enabled_v(); | ||
204 | } | ||
205 | |||
206 | void gm20b_fb_set_debug_mode(struct gk20a *g, bool enable) | ||
207 | { | ||
208 | u32 reg_val, fb_debug_ctrl, gpc_debug_ctrl; | ||
209 | |||
210 | if (enable) { | ||
211 | fb_debug_ctrl = fb_mmu_debug_ctrl_debug_enabled_f(); | ||
212 | gpc_debug_ctrl = gr_gpcs_pri_mmu_debug_ctrl_debug_enabled_f(); | ||
213 | g->mmu_debug_ctrl = true; | ||
214 | } else { | ||
215 | fb_debug_ctrl = fb_mmu_debug_ctrl_debug_disabled_f(); | ||
216 | gpc_debug_ctrl = gr_gpcs_pri_mmu_debug_ctrl_debug_disabled_f(); | ||
217 | g->mmu_debug_ctrl = false; | ||
218 | } | ||
219 | |||
220 | reg_val = gk20a_readl(g, fb_mmu_debug_ctrl_r()); | ||
221 | reg_val = set_field(reg_val, | ||
222 | fb_mmu_debug_ctrl_debug_m(), fb_debug_ctrl); | ||
223 | gk20a_writel(g, fb_mmu_debug_ctrl_r(), reg_val); | ||
224 | |||
225 | reg_val = gk20a_readl(g, gr_gpcs_pri_mmu_debug_ctrl_r()); | ||
226 | reg_val = set_field(reg_val, | ||
227 | gr_gpcs_pri_mmu_debug_ctrl_debug_m(), gpc_debug_ctrl); | ||
228 | gk20a_writel(g, gr_gpcs_pri_mmu_debug_ctrl_r(), reg_val); | ||
229 | } | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gm20b.h b/drivers/gpu/nvgpu/common/fb/fb_gm20b.h new file mode 100644 index 00000000..eb868b01 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gm20b.h | |||
@@ -0,0 +1,47 @@ | |||
1 | /* | ||
2 | * GM20B FB | ||
3 | * | ||
4 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef _NVHOST_GM20B_FB | ||
26 | #define _NVHOST_GM20B_FB | ||
27 | |||
28 | struct gk20a; | ||
29 | struct wpr_carveout_info; | ||
30 | |||
31 | void fb_gm20b_init_fs_state(struct gk20a *g); | ||
32 | void gm20b_fb_set_mmu_page_size(struct gk20a *g); | ||
33 | bool gm20b_fb_set_use_full_comp_tag_line(struct gk20a *g); | ||
34 | u32 gm20b_fb_mmu_ctrl(struct gk20a *g); | ||
35 | u32 gm20b_fb_mmu_debug_ctrl(struct gk20a *g); | ||
36 | u32 gm20b_fb_mmu_debug_wr(struct gk20a *g); | ||
37 | u32 gm20b_fb_mmu_debug_rd(struct gk20a *g); | ||
38 | unsigned int gm20b_fb_compression_page_size(struct gk20a *g); | ||
39 | unsigned int gm20b_fb_compressible_page_size(struct gk20a *g); | ||
40 | u32 gm20b_fb_compression_align_mask(struct gk20a *g); | ||
41 | void gm20b_fb_dump_vpr_wpr_info(struct gk20a *g); | ||
42 | void gm20b_fb_read_wpr_info(struct gk20a *g, struct wpr_carveout_info *inf); | ||
43 | int gm20b_fb_vpr_info_fetch(struct gk20a *g); | ||
44 | bool gm20b_fb_debug_mode_enabled(struct gk20a *g); | ||
45 | void gm20b_fb_set_debug_mode(struct gk20a *g, bool enable); | ||
46 | |||
47 | #endif | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gp106.c b/drivers/gpu/nvgpu/common/fb/fb_gp106.c new file mode 100644 index 00000000..6f257db4 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gp106.c | |||
@@ -0,0 +1,51 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #include "gk20a/gk20a.h" | ||
24 | |||
25 | #include "fb_gp10b.h" | ||
26 | #include "fb_gp106.h" | ||
27 | |||
28 | #include <nvgpu/hw/gp106/hw_fb_gp106.h> | ||
29 | |||
30 | #define HW_SCRUB_TIMEOUT_DEFAULT 100 /* usec */ | ||
31 | #define HW_SCRUB_TIMEOUT_MAX 2000000 /* usec */ | ||
32 | |||
33 | void gp106_fb_reset(struct gk20a *g) | ||
34 | { | ||
35 | u32 val; | ||
36 | |||
37 | int retries = HW_SCRUB_TIMEOUT_MAX / HW_SCRUB_TIMEOUT_DEFAULT; | ||
38 | /* wait for memory to be accessible */ | ||
39 | do { | ||
40 | u32 w = gk20a_readl(g, fb_niso_scrub_status_r()); | ||
41 | if (fb_niso_scrub_status_flag_v(w)) { | ||
42 | nvgpu_log_fn(g, "done"); | ||
43 | break; | ||
44 | } | ||
45 | nvgpu_udelay(HW_SCRUB_TIMEOUT_DEFAULT); | ||
46 | } while (--retries); | ||
47 | |||
48 | val = gk20a_readl(g, fb_mmu_priv_level_mask_r()); | ||
49 | val &= ~fb_mmu_priv_level_mask_write_violation_m(); | ||
50 | gk20a_writel(g, fb_mmu_priv_level_mask_r(), val); | ||
51 | } | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gp106.h b/drivers/gpu/nvgpu/common/fb/fb_gp106.h new file mode 100644 index 00000000..d5ee87f4 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gp106.h | |||
@@ -0,0 +1,28 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifndef FB_GP106_H | ||
24 | #define FB_GP106_H | ||
25 | struct gpu_ops; | ||
26 | |||
27 | void gp106_fb_reset(struct gk20a *g); | ||
28 | #endif | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gp10b.c b/drivers/gpu/nvgpu/common/fb/fb_gp10b.c new file mode 100644 index 00000000..45fc8373 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gp10b.c | |||
@@ -0,0 +1,38 @@ | |||
1 | /* | ||
2 | * GP10B FB | ||
3 | * | ||
4 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <nvgpu/sizes.h> | ||
26 | |||
27 | #include "gk20a/gk20a.h" | ||
28 | #include "fb_gp10b.h" | ||
29 | |||
30 | unsigned int gp10b_fb_compression_page_size(struct gk20a *g) | ||
31 | { | ||
32 | return SZ_64K; | ||
33 | } | ||
34 | |||
35 | unsigned int gp10b_fb_compressible_page_size(struct gk20a *g) | ||
36 | { | ||
37 | return SZ_4K; | ||
38 | } | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gp10b.h b/drivers/gpu/nvgpu/common/fb/fb_gp10b.h new file mode 100644 index 00000000..52aa2a75 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gp10b.h | |||
@@ -0,0 +1,32 @@ | |||
1 | /* | ||
2 | * GP10B FB | ||
3 | * | ||
4 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef _NVGPU_GP10B_FB | ||
26 | #define _NVGPU_GP10B_FB | ||
27 | struct gk20a; | ||
28 | |||
29 | unsigned int gp10b_fb_compression_page_size(struct gk20a *g); | ||
30 | unsigned int gp10b_fb_compressible_page_size(struct gk20a *g); | ||
31 | |||
32 | #endif | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gv100.c b/drivers/gpu/nvgpu/common/fb/fb_gv100.c new file mode 100644 index 00000000..155c1e8b --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gv100.c | |||
@@ -0,0 +1,257 @@ | |||
1 | /* | ||
2 | * GV100 FB | ||
3 | * | ||
4 | * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <nvgpu/types.h> | ||
26 | |||
27 | #include <nvgpu/dma.h> | ||
28 | #include <nvgpu/log.h> | ||
29 | #include <nvgpu/enabled.h> | ||
30 | #include <nvgpu/gmmu.h> | ||
31 | #include <nvgpu/nvgpu_common.h> | ||
32 | #include <nvgpu/kmem.h> | ||
33 | #include <nvgpu/nvgpu_mem.h> | ||
34 | #include <nvgpu/acr/nvgpu_acr.h> | ||
35 | #include <nvgpu/firmware.h> | ||
36 | #include <nvgpu/pmu.h> | ||
37 | #include <nvgpu/falcon.h> | ||
38 | |||
39 | #include "gk20a/gk20a.h" | ||
40 | #include "gm20b/acr_gm20b.h" | ||
41 | |||
42 | #include "fb_gv100.h" | ||
43 | |||
44 | #include <nvgpu/hw/gv100/hw_fb_gv100.h> | ||
45 | #include <nvgpu/hw/gv100/hw_falcon_gv100.h> | ||
46 | #include <nvgpu/hw/gv100/hw_mc_gv100.h> | ||
47 | |||
48 | #define HW_SCRUB_TIMEOUT_DEFAULT 100 /* usec */ | ||
49 | #define HW_SCRUB_TIMEOUT_MAX 2000000 /* usec */ | ||
50 | #define MEM_UNLOCK_TIMEOUT 3500 /* msec */ | ||
51 | |||
52 | void gv100_fb_reset(struct gk20a *g) | ||
53 | { | ||
54 | u32 val; | ||
55 | int retries = HW_SCRUB_TIMEOUT_MAX / HW_SCRUB_TIMEOUT_DEFAULT; | ||
56 | |||
57 | nvgpu_info(g, "reset gv100 fb"); | ||
58 | |||
59 | /* wait for memory to be accessible */ | ||
60 | do { | ||
61 | u32 w = gk20a_readl(g, fb_niso_scrub_status_r()); | ||
62 | if (fb_niso_scrub_status_flag_v(w)) { | ||
63 | nvgpu_info(g, "done"); | ||
64 | break; | ||
65 | } | ||
66 | nvgpu_udelay(HW_SCRUB_TIMEOUT_DEFAULT); | ||
67 | } while (--retries); | ||
68 | |||
69 | val = gk20a_readl(g, fb_mmu_priv_level_mask_r()); | ||
70 | val &= ~fb_mmu_priv_level_mask_write_violation_m(); | ||
71 | gk20a_writel(g, fb_mmu_priv_level_mask_r(), val); | ||
72 | } | ||
73 | |||
74 | void gv100_fb_enable_hub_intr(struct gk20a *g) | ||
75 | { | ||
76 | u32 mask = 0; | ||
77 | |||
78 | mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | | ||
79 | fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | | ||
80 | fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | | ||
81 | fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | | ||
82 | fb_niso_intr_en_set_mmu_replayable_fault_overflow_m(); | ||
83 | |||
84 | gk20a_writel(g, fb_niso_intr_en_set_r(0), | ||
85 | mask); | ||
86 | } | ||
87 | |||
88 | void gv100_fb_disable_hub_intr(struct gk20a *g) | ||
89 | { | ||
90 | u32 mask = 0; | ||
91 | |||
92 | mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | | ||
93 | fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | | ||
94 | fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | | ||
95 | fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | | ||
96 | fb_niso_intr_en_set_mmu_replayable_fault_overflow_m(); | ||
97 | |||
98 | gk20a_writel(g, fb_niso_intr_en_clr_r(0), | ||
99 | mask); | ||
100 | } | ||
101 | |||
102 | int gv100_fb_memory_unlock(struct gk20a *g) | ||
103 | { | ||
104 | struct nvgpu_firmware *mem_unlock_fw = NULL; | ||
105 | struct bin_hdr *hsbin_hdr = NULL; | ||
106 | struct acr_fw_header *fw_hdr = NULL; | ||
107 | u32 *mem_unlock_ucode = NULL; | ||
108 | u32 *mem_unlock_ucode_header = NULL; | ||
109 | u32 sec_imem_dest = 0; | ||
110 | u32 val = 0; | ||
111 | int err = 0; | ||
112 | |||
113 | nvgpu_log_fn(g, " "); | ||
114 | |||
115 | nvgpu_log_info(g, "fb_mmu_vpr_info = 0x%08x", | ||
116 | gk20a_readl(g, fb_mmu_vpr_info_r())); | ||
117 | /* | ||
118 | * mem_unlock.bin should be written to install | ||
119 | * traps even if VPR isn’t actually supported | ||
120 | */ | ||
121 | mem_unlock_fw = nvgpu_request_firmware(g, "mem_unlock.bin", 0); | ||
122 | if (!mem_unlock_fw) { | ||
123 | nvgpu_err(g, "mem unlock ucode get fail"); | ||
124 | err = -ENOENT; | ||
125 | goto exit; | ||
126 | } | ||
127 | |||
128 | /* Enable nvdec */ | ||
129 | g->ops.mc.enable(g, mc_enable_nvdec_enabled_f()); | ||
130 | |||
131 | /* nvdec falcon reset */ | ||
132 | nvgpu_flcn_reset(&g->nvdec_flcn); | ||
133 | |||
134 | hsbin_hdr = (struct bin_hdr *)mem_unlock_fw->data; | ||
135 | fw_hdr = (struct acr_fw_header *)(mem_unlock_fw->data + | ||
136 | hsbin_hdr->header_offset); | ||
137 | |||
138 | mem_unlock_ucode_header = (u32 *)(mem_unlock_fw->data + | ||
139 | fw_hdr->hdr_offset); | ||
140 | mem_unlock_ucode = (u32 *)(mem_unlock_fw->data + | ||
141 | hsbin_hdr->data_offset); | ||
142 | |||
143 | /* Patch Ucode singnatures */ | ||
144 | if (acr_ucode_patch_sig(g, mem_unlock_ucode, | ||
145 | (u32 *)(mem_unlock_fw->data + fw_hdr->sig_prod_offset), | ||
146 | (u32 *)(mem_unlock_fw->data + fw_hdr->sig_dbg_offset), | ||
147 | (u32 *)(mem_unlock_fw->data + fw_hdr->patch_loc), | ||
148 | (u32 *)(mem_unlock_fw->data + fw_hdr->patch_sig)) < 0) { | ||
149 | nvgpu_err(g, "mem unlock patch signatures fail"); | ||
150 | err = -EPERM; | ||
151 | goto exit; | ||
152 | } | ||
153 | |||
154 | /* Clear interrupts */ | ||
155 | nvgpu_flcn_set_irq(&g->nvdec_flcn, false, 0x0, 0x0); | ||
156 | |||
157 | /* Copy Non Secure IMEM code */ | ||
158 | nvgpu_flcn_copy_to_imem(&g->nvdec_flcn, 0, | ||
159 | (u8 *)&mem_unlock_ucode[ | ||
160 | mem_unlock_ucode_header[OS_CODE_OFFSET] >> 2], | ||
161 | mem_unlock_ucode_header[OS_CODE_SIZE], 0, false, | ||
162 | GET_IMEM_TAG(mem_unlock_ucode_header[OS_CODE_OFFSET])); | ||
163 | |||
164 | /* Put secure code after non-secure block */ | ||
165 | sec_imem_dest = GET_NEXT_BLOCK(mem_unlock_ucode_header[OS_CODE_SIZE]); | ||
166 | |||
167 | nvgpu_flcn_copy_to_imem(&g->nvdec_flcn, sec_imem_dest, | ||
168 | (u8 *)&mem_unlock_ucode[ | ||
169 | mem_unlock_ucode_header[APP_0_CODE_OFFSET] >> 2], | ||
170 | mem_unlock_ucode_header[APP_0_CODE_SIZE], 0, true, | ||
171 | GET_IMEM_TAG(mem_unlock_ucode_header[APP_0_CODE_OFFSET])); | ||
172 | |||
173 | /* load DMEM: ensure that signatures are patched */ | ||
174 | nvgpu_flcn_copy_to_dmem(&g->nvdec_flcn, 0, (u8 *)&mem_unlock_ucode[ | ||
175 | mem_unlock_ucode_header[OS_DATA_OFFSET] >> 2], | ||
176 | mem_unlock_ucode_header[OS_DATA_SIZE], 0); | ||
177 | |||
178 | nvgpu_log_info(g, "nvdec sctl reg %x\n", | ||
179 | gk20a_readl(g, g->nvdec_flcn.flcn_base + | ||
180 | falcon_falcon_sctl_r())); | ||
181 | |||
182 | /* set BOOTVEC to start of non-secure code */ | ||
183 | nvgpu_flcn_bootstrap(&g->nvdec_flcn, 0); | ||
184 | |||
185 | /* wait for complete & halt */ | ||
186 | nvgpu_flcn_wait_for_halt(&g->nvdec_flcn, MEM_UNLOCK_TIMEOUT); | ||
187 | |||
188 | /* check mem unlock status */ | ||
189 | val = nvgpu_flcn_mailbox_read(&g->nvdec_flcn, 0); | ||
190 | if (val) { | ||
191 | nvgpu_err(g, "memory unlock failed, err %x", val); | ||
192 | err = -1; | ||
193 | goto exit; | ||
194 | } | ||
195 | |||
196 | nvgpu_log_info(g, "nvdec sctl reg %x\n", | ||
197 | gk20a_readl(g, g->nvdec_flcn.flcn_base + | ||
198 | falcon_falcon_sctl_r())); | ||
199 | |||
200 | exit: | ||
201 | if (mem_unlock_fw) | ||
202 | nvgpu_release_firmware(g, mem_unlock_fw); | ||
203 | |||
204 | nvgpu_log_fn(g, "done, status - %d", err); | ||
205 | |||
206 | return err; | ||
207 | } | ||
208 | |||
209 | int gv100_fb_init_nvlink(struct gk20a *g) | ||
210 | { | ||
211 | u32 data; | ||
212 | u32 mask = g->nvlink.enabled_links; | ||
213 | |||
214 | /* Map enabled link to SYSMEM */ | ||
215 | data = nvgpu_readl(g, fb_hshub_config0_r()); | ||
216 | data = set_field(data, fb_hshub_config0_sysmem_nvlink_mask_m(), | ||
217 | fb_hshub_config0_sysmem_nvlink_mask_f(mask)); | ||
218 | nvgpu_writel(g, fb_hshub_config0_r(), data); | ||
219 | |||
220 | return 0; | ||
221 | } | ||
222 | |||
223 | int gv100_fb_enable_nvlink(struct gk20a *g) | ||
224 | { | ||
225 | u32 data; | ||
226 | |||
227 | nvgpu_log(g, gpu_dbg_nvlink|gpu_dbg_info, "enabling nvlink"); | ||
228 | |||
229 | /* Enable nvlink for NISO FBHUB */ | ||
230 | data = nvgpu_readl(g, fb_niso_cfg1_r()); | ||
231 | data = set_field(data, fb_niso_cfg1_sysmem_nvlink_m(), | ||
232 | fb_niso_cfg1_sysmem_nvlink_enabled_f()); | ||
233 | nvgpu_writel(g, fb_niso_cfg1_r(), data); | ||
234 | |||
235 | /* Setup atomics */ | ||
236 | data = nvgpu_readl(g, fb_mmu_ctrl_r()); | ||
237 | data = set_field(data, fb_mmu_ctrl_atomic_capability_mode_m(), | ||
238 | fb_mmu_ctrl_atomic_capability_mode_rmw_f()); | ||
239 | nvgpu_writel(g, fb_mmu_ctrl_r(), data); | ||
240 | |||
241 | data = nvgpu_readl(g, fb_hsmmu_pri_mmu_ctrl_r()); | ||
242 | data = set_field(data, fb_hsmmu_pri_mmu_ctrl_atomic_capability_mode_m(), | ||
243 | fb_hsmmu_pri_mmu_ctrl_atomic_capability_mode_rmw_f()); | ||
244 | nvgpu_writel(g, fb_hsmmu_pri_mmu_ctrl_r(), data); | ||
245 | |||
246 | data = nvgpu_readl(g, fb_fbhub_num_active_ltcs_r()); | ||
247 | data = set_field(data, fb_fbhub_num_active_ltcs_hub_sys_atomic_mode_m(), | ||
248 | fb_fbhub_num_active_ltcs_hub_sys_atomic_mode_use_rmw_f()); | ||
249 | nvgpu_writel(g, fb_fbhub_num_active_ltcs_r(), data); | ||
250 | |||
251 | data = nvgpu_readl(g, fb_hshub_num_active_ltcs_r()); | ||
252 | data = set_field(data, fb_hshub_num_active_ltcs_hub_sys_atomic_mode_m(), | ||
253 | fb_hshub_num_active_ltcs_hub_sys_atomic_mode_use_rmw_f()); | ||
254 | nvgpu_writel(g, fb_hshub_num_active_ltcs_r(), data); | ||
255 | |||
256 | return 0; | ||
257 | } | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gv100.h b/drivers/gpu/nvgpu/common/fb/fb_gv100.h new file mode 100644 index 00000000..195baccf --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gv100.h | |||
@@ -0,0 +1,36 @@ | |||
1 | /* | ||
2 | * GV100 FB | ||
3 | * | ||
4 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef _NVGPU_GV100_FB | ||
26 | #define _NVGPU_GV100_FB | ||
27 | |||
28 | struct gk20a; | ||
29 | |||
30 | void gv100_fb_reset(struct gk20a *g); | ||
31 | void gv100_fb_enable_hub_intr(struct gk20a *g); | ||
32 | void gv100_fb_disable_hub_intr(struct gk20a *g); | ||
33 | int gv100_fb_memory_unlock(struct gk20a *g); | ||
34 | int gv100_fb_init_nvlink(struct gk20a *g); | ||
35 | int gv100_fb_enable_nvlink(struct gk20a *g); | ||
36 | #endif | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gv11b.c b/drivers/gpu/nvgpu/common/fb/fb_gv11b.c new file mode 100644 index 00000000..69a71575 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gv11b.c | |||
@@ -0,0 +1,1516 @@ | |||
1 | /* | ||
2 | * GV11B FB | ||
3 | * | ||
4 | * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <nvgpu/dma.h> | ||
26 | #include <nvgpu/log.h> | ||
27 | #include <nvgpu/enabled.h> | ||
28 | #include <nvgpu/gmmu.h> | ||
29 | #include <nvgpu/barrier.h> | ||
30 | #include <nvgpu/bug.h> | ||
31 | #include <nvgpu/soc.h> | ||
32 | |||
33 | #include "gk20a/gk20a.h" | ||
34 | #include "gk20a/mm_gk20a.h" | ||
35 | |||
36 | #include "gv11b/fifo_gv11b.h" | ||
37 | #include "gv11b/ce_gv11b.h" | ||
38 | |||
39 | #include "fb_gk20a.h" | ||
40 | #include "fb_gp10b.h" | ||
41 | #include "fb_gv11b.h" | ||
42 | |||
43 | #include <nvgpu/hw/gv11b/hw_fb_gv11b.h> | ||
44 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
45 | #include <nvgpu/hw/gv11b/hw_ram_gv11b.h> | ||
46 | #include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h> | ||
47 | |||
48 | static int gv11b_fb_fix_page_fault(struct gk20a *g, | ||
49 | struct mmu_fault_info *mmfault); | ||
50 | |||
51 | static void gv11b_init_nvlink_soc_credits(struct gk20a *g) | ||
52 | { | ||
53 | if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) { | ||
54 | nvgpu_log(g, gpu_dbg_info, "nvlink soc credits init done by bpmp"); | ||
55 | } else { | ||
56 | #ifndef __NVGPU_POSIX__ | ||
57 | nvgpu_mss_nvlink_init_credits(g); | ||
58 | #endif | ||
59 | } | ||
60 | } | ||
61 | |||
62 | void gv11b_fb_init_hw(struct gk20a *g) | ||
63 | { | ||
64 | gk20a_fb_init_hw(g); | ||
65 | |||
66 | g->ops.fb.enable_hub_intr(g); | ||
67 | } | ||
68 | |||
69 | void gv11b_fb_init_fs_state(struct gk20a *g) | ||
70 | { | ||
71 | nvgpu_log(g, gpu_dbg_fn, "initialize gv11b fb"); | ||
72 | |||
73 | nvgpu_log(g, gpu_dbg_info, "fbhub active ltcs %x", | ||
74 | gk20a_readl(g, fb_fbhub_num_active_ltcs_r())); | ||
75 | |||
76 | nvgpu_log(g, gpu_dbg_info, "mmu active ltcs %u", | ||
77 | fb_mmu_num_active_ltcs_count_v( | ||
78 | gk20a_readl(g, fb_mmu_num_active_ltcs_r()))); | ||
79 | |||
80 | if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) { | ||
81 | /* Bypass MMU check for non-secure boot. For | ||
82 | * secure-boot,this register write has no-effect */ | ||
83 | gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffffU); | ||
84 | } | ||
85 | } | ||
86 | |||
87 | void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr) | ||
88 | { | ||
89 | u32 max_size = gr->max_comptag_mem; | ||
90 | /* one tag line covers 64KB */ | ||
91 | u32 max_comptag_lines = max_size << 4; | ||
92 | u32 compbit_base_post_divide; | ||
93 | u64 compbit_base_post_multiply64; | ||
94 | u64 compbit_store_iova; | ||
95 | u64 compbit_base_post_divide64; | ||
96 | |||
97 | if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) | ||
98 | compbit_store_iova = nvgpu_mem_get_phys_addr(g, | ||
99 | &gr->compbit_store.mem); | ||
100 | else | ||
101 | compbit_store_iova = nvgpu_mem_get_addr(g, | ||
102 | &gr->compbit_store.mem); | ||
103 | |||
104 | compbit_base_post_divide64 = compbit_store_iova >> | ||
105 | fb_mmu_cbc_base_address_alignment_shift_v(); | ||
106 | |||
107 | do_div(compbit_base_post_divide64, g->ltc_count); | ||
108 | compbit_base_post_divide = u64_lo32(compbit_base_post_divide64); | ||
109 | |||
110 | compbit_base_post_multiply64 = ((u64)compbit_base_post_divide * | ||
111 | g->ltc_count) << fb_mmu_cbc_base_address_alignment_shift_v(); | ||
112 | |||
113 | if (compbit_base_post_multiply64 < compbit_store_iova) | ||
114 | compbit_base_post_divide++; | ||
115 | |||
116 | if (g->ops.ltc.cbc_fix_config) | ||
117 | compbit_base_post_divide = | ||
118 | g->ops.ltc.cbc_fix_config(g, compbit_base_post_divide); | ||
119 | |||
120 | gk20a_writel(g, fb_mmu_cbc_base_r(), | ||
121 | fb_mmu_cbc_base_address_f(compbit_base_post_divide)); | ||
122 | |||
123 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte, | ||
124 | "compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n", | ||
125 | (u32)(compbit_store_iova >> 32), | ||
126 | (u32)(compbit_store_iova & 0xffffffff), | ||
127 | compbit_base_post_divide); | ||
128 | nvgpu_log(g, gpu_dbg_fn, "cbc base %x", | ||
129 | gk20a_readl(g, fb_mmu_cbc_base_r())); | ||
130 | |||
131 | gr->compbit_store.base_hw = compbit_base_post_divide; | ||
132 | |||
133 | g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_invalidate, | ||
134 | 0, max_comptag_lines - 1); | ||
135 | |||
136 | } | ||
137 | |||
138 | void gv11b_fb_reset(struct gk20a *g) | ||
139 | { | ||
140 | gv11b_init_nvlink_soc_credits(g); | ||
141 | } | ||
142 | |||
143 | static const char * const invalid_str = "invalid"; | ||
144 | |||
145 | static const char *const fault_type_descs_gv11b[] = { | ||
146 | "invalid pde", | ||
147 | "invalid pde size", | ||
148 | "invalid pte", | ||
149 | "limit violation", | ||
150 | "unbound inst block", | ||
151 | "priv violation", | ||
152 | "write", | ||
153 | "read", | ||
154 | "pitch mask violation", | ||
155 | "work creation", | ||
156 | "unsupported aperture", | ||
157 | "compression failure", | ||
158 | "unsupported kind", | ||
159 | "region violation", | ||
160 | "poison", | ||
161 | "atomic" | ||
162 | }; | ||
163 | |||
164 | static const char *const fault_client_type_descs_gv11b[] = { | ||
165 | "gpc", | ||
166 | "hub", | ||
167 | }; | ||
168 | |||
169 | static const char *const fault_access_type_descs_gv11b[] = { | ||
170 | "virt read", | ||
171 | "virt write", | ||
172 | "virt atomic strong", | ||
173 | "virt prefetch", | ||
174 | "virt atomic weak", | ||
175 | "xxx", | ||
176 | "xxx", | ||
177 | "xxx", | ||
178 | "phys read", | ||
179 | "phys write", | ||
180 | "phys atomic", | ||
181 | "phys prefetch", | ||
182 | }; | ||
183 | |||
184 | static const char *const hub_client_descs_gv11b[] = { | ||
185 | "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu", | ||
186 | "host cpu nb", "iso", "mmu", "nvdec", "nvenc1", "nvenc2", | ||
187 | "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc", | ||
188 | "scc nb", "sec", "ssync", "gr copy", "xv", "mmu nb", | ||
189 | "nvenc", "d falcon", "sked", "a falcon", "hsce0", "hsce1", | ||
190 | "hsce2", "hsce3", "hsce4", "hsce5", "hsce6", "hsce7", "hsce8", | ||
191 | "hsce9", "hshub", "ptp x0", "ptp x1", "ptp x2", "ptp x3", | ||
192 | "ptp x4", "ptp x5", "ptp x6", "ptp x7", "vpr scrubber0", | ||
193 | "vpr scrubber1", "dwbif", "fbfalcon", "ce shim", "gsp", | ||
194 | "dont care" | ||
195 | }; | ||
196 | |||
197 | static const char *const gpc_client_descs_gv11b[] = { | ||
198 | "t1 0", "t1 1", "t1 2", "t1 3", | ||
199 | "t1 4", "t1 5", "t1 6", "t1 7", | ||
200 | "pe 0", "pe 1", "pe 2", "pe 3", | ||
201 | "pe 4", "pe 5", "pe 6", "pe 7", | ||
202 | "rast", "gcc", "gpccs", | ||
203 | "prop 0", "prop 1", "prop 2", "prop 3", | ||
204 | "gpm", | ||
205 | "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3", | ||
206 | "ltp utlb 4", "ltp utlb 5", "ltp utlb 6", "ltp utlb 7", | ||
207 | "utlb", | ||
208 | "t1 8", "t1 9", "t1 10", "t1 11", | ||
209 | "t1 12", "t1 13", "t1 14", "t1 15", | ||
210 | "tpccs 0", "tpccs 1", "tpccs 2", "tpccs 3", | ||
211 | "tpccs 4", "tpccs 5", "tpccs 6", "tpccs 7", | ||
212 | "pe 8", "pe 9", "tpccs 8", "tpccs 9", | ||
213 | "t1 16", "t1 17", "t1 18", "t1 19", | ||
214 | "pe 10", "pe 11", "tpccs 10", "tpccs 11", | ||
215 | "t1 20", "t1 21", "t1 22", "t1 23", | ||
216 | "pe 12", "pe 13", "tpccs 12", "tpccs 13", | ||
217 | "t1 24", "t1 25", "t1 26", "t1 27", | ||
218 | "pe 14", "pe 15", "tpccs 14", "tpccs 15", | ||
219 | "t1 28", "t1 29", "t1 30", "t1 31", | ||
220 | "pe 16", "pe 17", "tpccs 16", "tpccs 17", | ||
221 | "t1 32", "t1 33", "t1 34", "t1 35", | ||
222 | "pe 18", "pe 19", "tpccs 18", "tpccs 19", | ||
223 | "t1 36", "t1 37", "t1 38", "t1 39", | ||
224 | }; | ||
225 | |||
226 | bool gv11b_fb_is_fault_buf_enabled(struct gk20a *g, u32 index) | ||
227 | { | ||
228 | u32 reg_val; | ||
229 | |||
230 | reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); | ||
231 | return fb_mmu_fault_buffer_size_enable_v(reg_val) != 0U; | ||
232 | } | ||
233 | |||
234 | static void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g, | ||
235 | u32 index, u32 next) | ||
236 | { | ||
237 | u32 reg_val; | ||
238 | |||
239 | nvgpu_log(g, gpu_dbg_intr, "updating get index with = %d", next); | ||
240 | |||
241 | reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); | ||
242 | reg_val = set_field(reg_val, fb_mmu_fault_buffer_get_ptr_m(), | ||
243 | fb_mmu_fault_buffer_get_ptr_f(next)); | ||
244 | |||
245 | /* while the fault is being handled it is possible for overflow | ||
246 | * to happen, | ||
247 | */ | ||
248 | if (reg_val & fb_mmu_fault_buffer_get_overflow_m()) | ||
249 | reg_val |= fb_mmu_fault_buffer_get_overflow_clear_f(); | ||
250 | |||
251 | g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); | ||
252 | |||
253 | /* make sure get ptr update is visible to everyone to avoid | ||
254 | * reading already read entry | ||
255 | */ | ||
256 | nvgpu_mb(); | ||
257 | } | ||
258 | |||
259 | static u32 gv11b_fb_fault_buffer_get_index(struct gk20a *g, u32 index) | ||
260 | { | ||
261 | u32 reg_val; | ||
262 | |||
263 | reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); | ||
264 | return fb_mmu_fault_buffer_get_ptr_v(reg_val); | ||
265 | } | ||
266 | |||
267 | static u32 gv11b_fb_fault_buffer_put_index(struct gk20a *g, u32 index) | ||
268 | { | ||
269 | u32 reg_val; | ||
270 | |||
271 | reg_val = g->ops.fb.read_mmu_fault_buffer_put(g, index); | ||
272 | return fb_mmu_fault_buffer_put_ptr_v(reg_val); | ||
273 | } | ||
274 | |||
275 | static u32 gv11b_fb_fault_buffer_size_val(struct gk20a *g, u32 index) | ||
276 | { | ||
277 | u32 reg_val; | ||
278 | |||
279 | reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); | ||
280 | return fb_mmu_fault_buffer_size_val_v(reg_val); | ||
281 | } | ||
282 | |||
283 | static bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g, | ||
284 | u32 index, u32 *get_idx) | ||
285 | { | ||
286 | u32 put_idx; | ||
287 | |||
288 | *get_idx = gv11b_fb_fault_buffer_get_index(g, index); | ||
289 | put_idx = gv11b_fb_fault_buffer_put_index(g, index); | ||
290 | |||
291 | return *get_idx == put_idx; | ||
292 | } | ||
293 | |||
294 | static bool gv11b_fb_is_fault_buffer_full(struct gk20a *g, u32 index) | ||
295 | { | ||
296 | u32 get_idx, put_idx, entries; | ||
297 | |||
298 | |||
299 | get_idx = gv11b_fb_fault_buffer_get_index(g, index); | ||
300 | |||
301 | put_idx = gv11b_fb_fault_buffer_put_index(g, index); | ||
302 | |||
303 | entries = gv11b_fb_fault_buffer_size_val(g, index); | ||
304 | |||
305 | return get_idx == ((put_idx + 1) % entries); | ||
306 | } | ||
307 | |||
308 | void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, | ||
309 | u32 index, u32 state) | ||
310 | { | ||
311 | u32 fault_status; | ||
312 | u32 reg_val; | ||
313 | |||
314 | nvgpu_log_fn(g, " "); | ||
315 | |||
316 | reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); | ||
317 | if (state == NVGPU_FB_MMU_FAULT_BUF_ENABLED) { | ||
318 | if (gv11b_fb_is_fault_buf_enabled(g, index)) { | ||
319 | nvgpu_log_info(g, "fault buffer is already enabled"); | ||
320 | } else { | ||
321 | reg_val |= fb_mmu_fault_buffer_size_enable_true_f(); | ||
322 | g->ops.fb.write_mmu_fault_buffer_size(g, index, | ||
323 | reg_val); | ||
324 | } | ||
325 | |||
326 | } else { | ||
327 | struct nvgpu_timeout timeout; | ||
328 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
329 | |||
330 | nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), | ||
331 | NVGPU_TIMER_CPU_TIMER); | ||
332 | |||
333 | reg_val &= (~(fb_mmu_fault_buffer_size_enable_m())); | ||
334 | g->ops.fb.write_mmu_fault_buffer_size(g, index, reg_val); | ||
335 | |||
336 | fault_status = g->ops.fb.read_mmu_fault_status(g); | ||
337 | |||
338 | do { | ||
339 | if (!(fault_status & fb_mmu_fault_status_busy_true_f())) | ||
340 | break; | ||
341 | /* | ||
342 | * Make sure fault buffer is disabled. | ||
343 | * This is to avoid accessing fault buffer by hw | ||
344 | * during the window BAR2 is being unmapped by s/w | ||
345 | */ | ||
346 | nvgpu_log_info(g, "fault status busy set, check again"); | ||
347 | fault_status = g->ops.fb.read_mmu_fault_status(g); | ||
348 | |||
349 | nvgpu_usleep_range(delay, delay * 2); | ||
350 | delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); | ||
351 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
352 | "fault status busy set")); | ||
353 | } | ||
354 | } | ||
355 | |||
356 | void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, u32 index) | ||
357 | { | ||
358 | u32 addr_lo; | ||
359 | u32 addr_hi; | ||
360 | |||
361 | nvgpu_log_fn(g, " "); | ||
362 | |||
363 | gv11b_fb_fault_buf_set_state_hw(g, index, | ||
364 | NVGPU_FB_MMU_FAULT_BUF_DISABLED); | ||
365 | addr_lo = u64_lo32(g->mm.hw_fault_buf[index].gpu_va >> | ||
366 | ram_in_base_shift_v()); | ||
367 | addr_hi = u64_hi32(g->mm.hw_fault_buf[index].gpu_va); | ||
368 | |||
369 | g->ops.fb.write_mmu_fault_buffer_lo_hi(g, index, | ||
370 | fb_mmu_fault_buffer_lo_addr_f(addr_lo), | ||
371 | fb_mmu_fault_buffer_hi_addr_f(addr_hi)); | ||
372 | |||
373 | g->ops.fb.write_mmu_fault_buffer_size(g, index, | ||
374 | fb_mmu_fault_buffer_size_val_f(g->ops.fifo.get_num_fifos(g)) | | ||
375 | fb_mmu_fault_buffer_size_overflow_intr_enable_f()); | ||
376 | |||
377 | gv11b_fb_fault_buf_set_state_hw(g, index, NVGPU_FB_MMU_FAULT_BUF_ENABLED); | ||
378 | } | ||
379 | |||
380 | void gv11b_fb_enable_hub_intr(struct gk20a *g) | ||
381 | { | ||
382 | u32 mask = 0; | ||
383 | |||
384 | mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | | ||
385 | fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | | ||
386 | fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | | ||
387 | fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | | ||
388 | fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() | | ||
389 | fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); | ||
390 | |||
391 | gk20a_writel(g, fb_niso_intr_en_set_r(0), | ||
392 | mask); | ||
393 | } | ||
394 | |||
395 | void gv11b_fb_disable_hub_intr(struct gk20a *g) | ||
396 | { | ||
397 | u32 mask = 0; | ||
398 | |||
399 | mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | | ||
400 | fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | | ||
401 | fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | | ||
402 | fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | | ||
403 | fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() | | ||
404 | fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); | ||
405 | |||
406 | gk20a_writel(g, fb_niso_intr_en_clr_r(0), | ||
407 | mask); | ||
408 | } | ||
409 | |||
410 | void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
411 | { | ||
412 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
413 | u32 corrected_delta, uncorrected_delta; | ||
414 | u32 corrected_overflow, uncorrected_overflow; | ||
415 | |||
416 | ecc_addr = gk20a_readl(g, fb_mmu_l2tlb_ecc_address_r()); | ||
417 | corrected_cnt = gk20a_readl(g, | ||
418 | fb_mmu_l2tlb_ecc_corrected_err_count_r()); | ||
419 | uncorrected_cnt = gk20a_readl(g, | ||
420 | fb_mmu_l2tlb_ecc_uncorrected_err_count_r()); | ||
421 | |||
422 | corrected_delta = fb_mmu_l2tlb_ecc_corrected_err_count_total_v( | ||
423 | corrected_cnt); | ||
424 | uncorrected_delta = fb_mmu_l2tlb_ecc_uncorrected_err_count_total_v( | ||
425 | uncorrected_cnt); | ||
426 | corrected_overflow = ecc_status & | ||
427 | fb_mmu_l2tlb_ecc_status_corrected_err_total_counter_overflow_m(); | ||
428 | |||
429 | uncorrected_overflow = ecc_status & | ||
430 | fb_mmu_l2tlb_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
431 | |||
432 | /* clear the interrupt */ | ||
433 | if ((corrected_delta > 0) || corrected_overflow) | ||
434 | gk20a_writel(g, fb_mmu_l2tlb_ecc_corrected_err_count_r(), 0); | ||
435 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
436 | gk20a_writel(g, fb_mmu_l2tlb_ecc_uncorrected_err_count_r(), 0); | ||
437 | |||
438 | gk20a_writel(g, fb_mmu_l2tlb_ecc_status_r(), | ||
439 | fb_mmu_l2tlb_ecc_status_reset_clear_f()); | ||
440 | |||
441 | /* Handle overflow */ | ||
442 | if (corrected_overflow) | ||
443 | corrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_corrected_err_count_total_s()); | ||
444 | if (uncorrected_overflow) | ||
445 | uncorrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_uncorrected_err_count_total_s()); | ||
446 | |||
447 | |||
448 | g->ecc.fb.mmu_l2tlb_corrected_err_count.counters[0] += | ||
449 | corrected_delta; | ||
450 | g->ecc.fb.mmu_l2tlb_uncorrected_err_count.counters[0] += | ||
451 | uncorrected_delta; | ||
452 | |||
453 | if (ecc_status & fb_mmu_l2tlb_ecc_status_corrected_err_l2tlb_sa_data_m()) | ||
454 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); | ||
455 | if (ecc_status & fb_mmu_l2tlb_ecc_status_uncorrected_err_l2tlb_sa_data_m()) | ||
456 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); | ||
457 | if (corrected_overflow || uncorrected_overflow) | ||
458 | nvgpu_info(g, "mmu l2tlb ecc counter overflow!"); | ||
459 | |||
460 | nvgpu_log(g, gpu_dbg_intr, | ||
461 | "ecc error address: 0x%x", ecc_addr); | ||
462 | nvgpu_log(g, gpu_dbg_intr, | ||
463 | "ecc error count corrected: %d, uncorrected %d", | ||
464 | g->ecc.fb.mmu_l2tlb_corrected_err_count.counters[0], | ||
465 | g->ecc.fb.mmu_l2tlb_uncorrected_err_count.counters[0]); | ||
466 | } | ||
467 | |||
468 | void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
469 | { | ||
470 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
471 | u32 corrected_delta, uncorrected_delta; | ||
472 | u32 corrected_overflow, uncorrected_overflow; | ||
473 | |||
474 | ecc_addr = gk20a_readl(g, fb_mmu_hubtlb_ecc_address_r()); | ||
475 | corrected_cnt = gk20a_readl(g, | ||
476 | fb_mmu_hubtlb_ecc_corrected_err_count_r()); | ||
477 | uncorrected_cnt = gk20a_readl(g, | ||
478 | fb_mmu_hubtlb_ecc_uncorrected_err_count_r()); | ||
479 | |||
480 | corrected_delta = fb_mmu_hubtlb_ecc_corrected_err_count_total_v( | ||
481 | corrected_cnt); | ||
482 | uncorrected_delta = fb_mmu_hubtlb_ecc_uncorrected_err_count_total_v( | ||
483 | uncorrected_cnt); | ||
484 | corrected_overflow = ecc_status & | ||
485 | fb_mmu_hubtlb_ecc_status_corrected_err_total_counter_overflow_m(); | ||
486 | |||
487 | uncorrected_overflow = ecc_status & | ||
488 | fb_mmu_hubtlb_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
489 | |||
490 | /* clear the interrupt */ | ||
491 | if ((corrected_delta > 0) || corrected_overflow) | ||
492 | gk20a_writel(g, fb_mmu_hubtlb_ecc_corrected_err_count_r(), 0); | ||
493 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
494 | gk20a_writel(g, fb_mmu_hubtlb_ecc_uncorrected_err_count_r(), 0); | ||
495 | |||
496 | gk20a_writel(g, fb_mmu_hubtlb_ecc_status_r(), | ||
497 | fb_mmu_hubtlb_ecc_status_reset_clear_f()); | ||
498 | |||
499 | /* Handle overflow */ | ||
500 | if (corrected_overflow) | ||
501 | corrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_corrected_err_count_total_s()); | ||
502 | if (uncorrected_overflow) | ||
503 | uncorrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_uncorrected_err_count_total_s()); | ||
504 | |||
505 | |||
506 | g->ecc.fb.mmu_hubtlb_corrected_err_count.counters[0] += | ||
507 | corrected_delta; | ||
508 | g->ecc.fb.mmu_hubtlb_uncorrected_err_count.counters[0] += | ||
509 | uncorrected_delta; | ||
510 | |||
511 | if (ecc_status & fb_mmu_hubtlb_ecc_status_corrected_err_sa_data_m()) | ||
512 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); | ||
513 | if (ecc_status & fb_mmu_hubtlb_ecc_status_uncorrected_err_sa_data_m()) | ||
514 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); | ||
515 | if (corrected_overflow || uncorrected_overflow) | ||
516 | nvgpu_info(g, "mmu hubtlb ecc counter overflow!"); | ||
517 | |||
518 | nvgpu_log(g, gpu_dbg_intr, | ||
519 | "ecc error address: 0x%x", ecc_addr); | ||
520 | nvgpu_log(g, gpu_dbg_intr, | ||
521 | "ecc error count corrected: %d, uncorrected %d", | ||
522 | g->ecc.fb.mmu_hubtlb_corrected_err_count.counters[0], | ||
523 | g->ecc.fb.mmu_hubtlb_uncorrected_err_count.counters[0]); | ||
524 | } | ||
525 | |||
526 | void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
527 | { | ||
528 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
529 | u32 corrected_delta, uncorrected_delta; | ||
530 | u32 corrected_overflow, uncorrected_overflow; | ||
531 | |||
532 | ecc_addr = gk20a_readl(g, fb_mmu_fillunit_ecc_address_r()); | ||
533 | corrected_cnt = gk20a_readl(g, | ||
534 | fb_mmu_fillunit_ecc_corrected_err_count_r()); | ||
535 | uncorrected_cnt = gk20a_readl(g, | ||
536 | fb_mmu_fillunit_ecc_uncorrected_err_count_r()); | ||
537 | |||
538 | corrected_delta = fb_mmu_fillunit_ecc_corrected_err_count_total_v( | ||
539 | corrected_cnt); | ||
540 | uncorrected_delta = fb_mmu_fillunit_ecc_uncorrected_err_count_total_v( | ||
541 | uncorrected_cnt); | ||
542 | corrected_overflow = ecc_status & | ||
543 | fb_mmu_fillunit_ecc_status_corrected_err_total_counter_overflow_m(); | ||
544 | |||
545 | uncorrected_overflow = ecc_status & | ||
546 | fb_mmu_fillunit_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
547 | |||
548 | /* clear the interrupt */ | ||
549 | if ((corrected_delta > 0) || corrected_overflow) | ||
550 | gk20a_writel(g, fb_mmu_fillunit_ecc_corrected_err_count_r(), 0); | ||
551 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
552 | gk20a_writel(g, fb_mmu_fillunit_ecc_uncorrected_err_count_r(), 0); | ||
553 | |||
554 | gk20a_writel(g, fb_mmu_fillunit_ecc_status_r(), | ||
555 | fb_mmu_fillunit_ecc_status_reset_clear_f()); | ||
556 | |||
557 | /* Handle overflow */ | ||
558 | if (corrected_overflow) | ||
559 | corrected_delta += (0x1UL << fb_mmu_fillunit_ecc_corrected_err_count_total_s()); | ||
560 | if (uncorrected_overflow) | ||
561 | uncorrected_delta += (0x1UL << fb_mmu_fillunit_ecc_uncorrected_err_count_total_s()); | ||
562 | |||
563 | |||
564 | g->ecc.fb.mmu_fillunit_corrected_err_count.counters[0] += | ||
565 | corrected_delta; | ||
566 | g->ecc.fb.mmu_fillunit_uncorrected_err_count.counters[0] += | ||
567 | uncorrected_delta; | ||
568 | |||
569 | if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pte_data_m()) | ||
570 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc pte data error"); | ||
571 | if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pte_data_m()) | ||
572 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pte data error"); | ||
573 | if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pde0_data_m()) | ||
574 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc pde0 data error"); | ||
575 | if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pde0_data_m()) | ||
576 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pde0 data error"); | ||
577 | |||
578 | if (corrected_overflow || uncorrected_overflow) | ||
579 | nvgpu_info(g, "mmu fillunit ecc counter overflow!"); | ||
580 | |||
581 | nvgpu_log(g, gpu_dbg_intr, | ||
582 | "ecc error address: 0x%x", ecc_addr); | ||
583 | nvgpu_log(g, gpu_dbg_intr, | ||
584 | "ecc error count corrected: %d, uncorrected %d", | ||
585 | g->ecc.fb.mmu_fillunit_corrected_err_count.counters[0], | ||
586 | g->ecc.fb.mmu_fillunit_uncorrected_err_count.counters[0]); | ||
587 | } | ||
588 | |||
589 | static void gv11b_fb_parse_mmfault(struct mmu_fault_info *mmfault) | ||
590 | { | ||
591 | if (WARN_ON(mmfault->fault_type >= | ||
592 | ARRAY_SIZE(fault_type_descs_gv11b))) | ||
593 | mmfault->fault_type_desc = invalid_str; | ||
594 | else | ||
595 | mmfault->fault_type_desc = | ||
596 | fault_type_descs_gv11b[mmfault->fault_type]; | ||
597 | |||
598 | if (WARN_ON(mmfault->client_type >= | ||
599 | ARRAY_SIZE(fault_client_type_descs_gv11b))) | ||
600 | mmfault->client_type_desc = invalid_str; | ||
601 | else | ||
602 | mmfault->client_type_desc = | ||
603 | fault_client_type_descs_gv11b[mmfault->client_type]; | ||
604 | |||
605 | mmfault->client_id_desc = invalid_str; | ||
606 | if (mmfault->client_type == | ||
607 | gmmu_fault_client_type_hub_v()) { | ||
608 | |||
609 | if (!(WARN_ON(mmfault->client_id >= | ||
610 | ARRAY_SIZE(hub_client_descs_gv11b)))) | ||
611 | mmfault->client_id_desc = | ||
612 | hub_client_descs_gv11b[mmfault->client_id]; | ||
613 | } else if (mmfault->client_type == | ||
614 | gmmu_fault_client_type_gpc_v()) { | ||
615 | if (!(WARN_ON(mmfault->client_id >= | ||
616 | ARRAY_SIZE(gpc_client_descs_gv11b)))) | ||
617 | mmfault->client_id_desc = | ||
618 | gpc_client_descs_gv11b[mmfault->client_id]; | ||
619 | } | ||
620 | |||
621 | } | ||
622 | |||
623 | static void gv11b_fb_print_fault_info(struct gk20a *g, | ||
624 | struct mmu_fault_info *mmfault) | ||
625 | { | ||
626 | if (mmfault && mmfault->valid) { | ||
627 | nvgpu_err(g, "[MMU FAULT] " | ||
628 | "mmu engine id: %d, " | ||
629 | "ch id: %d, " | ||
630 | "fault addr: 0x%llx, " | ||
631 | "fault addr aperture: %d, " | ||
632 | "fault type: %s, " | ||
633 | "access type: %s, ", | ||
634 | mmfault->mmu_engine_id, | ||
635 | mmfault->chid, | ||
636 | mmfault->fault_addr, | ||
637 | mmfault->fault_addr_aperture, | ||
638 | mmfault->fault_type_desc, | ||
639 | fault_access_type_descs_gv11b[mmfault->access_type]); | ||
640 | nvgpu_err(g, "[MMU FAULT] " | ||
641 | "protected mode: %d, " | ||
642 | "client type: %s, " | ||
643 | "client id: %s, " | ||
644 | "gpc id if client type is gpc: %d, ", | ||
645 | mmfault->protected_mode, | ||
646 | mmfault->client_type_desc, | ||
647 | mmfault->client_id_desc, | ||
648 | mmfault->gpc_id); | ||
649 | |||
650 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
651 | "faulted act eng id if any: 0x%x, " | ||
652 | "faulted veid if any: 0x%x, " | ||
653 | "faulted pbdma id if any: 0x%x, ", | ||
654 | mmfault->faulted_engine, | ||
655 | mmfault->faulted_subid, | ||
656 | mmfault->faulted_pbdma); | ||
657 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
658 | "inst ptr: 0x%llx, " | ||
659 | "inst ptr aperture: %d, " | ||
660 | "replayable fault: %d, " | ||
661 | "replayable fault en: %d " | ||
662 | "timestamp hi:lo 0x%08x:0x%08x, ", | ||
663 | mmfault->inst_ptr, | ||
664 | mmfault->inst_aperture, | ||
665 | mmfault->replayable_fault, | ||
666 | mmfault->replay_fault_en, | ||
667 | mmfault->timestamp_hi, mmfault->timestamp_lo); | ||
668 | } | ||
669 | } | ||
670 | |||
671 | /* | ||
672 | *Fault buffer format | ||
673 | * | ||
674 | * 31 28 24 23 16 15 8 7 4 0 | ||
675 | *.-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. | ||
676 | *| inst_lo |0 0|apr|0 0 0 0 0 0 0 0| | ||
677 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
678 | *| inst_hi | | ||
679 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
680 | *| addr_31_12 | |AP | | ||
681 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
682 | *| addr_63_32 | | ||
683 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
684 | *| timestamp_lo | | ||
685 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
686 | *| timestamp_hi | | ||
687 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
688 | *| (reserved) | engine_id | | ||
689 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
690 | *|V|R|P| gpc_id |0 0 0|t|0|acctp|0| client |RF0 0|faulttype| | ||
691 | */ | ||
692 | |||
693 | static void gv11b_fb_copy_from_hw_fault_buf(struct gk20a *g, | ||
694 | struct nvgpu_mem *mem, u32 offset, struct mmu_fault_info *mmfault) | ||
695 | { | ||
696 | u32 rd32_val; | ||
697 | u32 addr_lo, addr_hi; | ||
698 | u64 inst_ptr; | ||
699 | u32 chid = FIFO_INVAL_CHANNEL_ID; | ||
700 | struct channel_gk20a *refch; | ||
701 | |||
702 | memset(mmfault, 0, sizeof(*mmfault)); | ||
703 | |||
704 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
705 | gmmu_fault_buf_entry_inst_lo_w()); | ||
706 | addr_lo = gmmu_fault_buf_entry_inst_lo_v(rd32_val); | ||
707 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
708 | |||
709 | addr_hi = nvgpu_mem_rd32(g, mem, offset + | ||
710 | gmmu_fault_buf_entry_inst_hi_w()); | ||
711 | addr_hi = gmmu_fault_buf_entry_inst_hi_v(addr_hi); | ||
712 | |||
713 | inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
714 | |||
715 | /* refch will be put back after fault is handled */ | ||
716 | refch = gk20a_refch_from_inst_ptr(g, inst_ptr); | ||
717 | if (refch) | ||
718 | chid = refch->chid; | ||
719 | |||
720 | /* it is ok to continue even if refch is NULL */ | ||
721 | mmfault->refch = refch; | ||
722 | mmfault->chid = chid; | ||
723 | mmfault->inst_ptr = inst_ptr; | ||
724 | mmfault->inst_aperture = gmmu_fault_buf_entry_inst_aperture_v(rd32_val); | ||
725 | |||
726 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
727 | gmmu_fault_buf_entry_addr_lo_w()); | ||
728 | |||
729 | mmfault->fault_addr_aperture = | ||
730 | gmmu_fault_buf_entry_addr_phys_aperture_v(rd32_val); | ||
731 | addr_lo = gmmu_fault_buf_entry_addr_lo_v(rd32_val); | ||
732 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
733 | |||
734 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
735 | gmmu_fault_buf_entry_addr_hi_w()); | ||
736 | addr_hi = gmmu_fault_buf_entry_addr_hi_v(rd32_val); | ||
737 | mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
738 | |||
739 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
740 | gmmu_fault_buf_entry_timestamp_lo_w()); | ||
741 | mmfault->timestamp_lo = | ||
742 | gmmu_fault_buf_entry_timestamp_lo_v(rd32_val); | ||
743 | |||
744 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
745 | gmmu_fault_buf_entry_timestamp_hi_w()); | ||
746 | mmfault->timestamp_hi = | ||
747 | gmmu_fault_buf_entry_timestamp_hi_v(rd32_val); | ||
748 | |||
749 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
750 | gmmu_fault_buf_entry_engine_id_w()); | ||
751 | |||
752 | mmfault->mmu_engine_id = | ||
753 | gmmu_fault_buf_entry_engine_id_v(rd32_val); | ||
754 | gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, | ||
755 | &mmfault->faulted_engine, &mmfault->faulted_subid, | ||
756 | &mmfault->faulted_pbdma); | ||
757 | |||
758 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
759 | gmmu_fault_buf_entry_fault_type_w()); | ||
760 | mmfault->client_id = | ||
761 | gmmu_fault_buf_entry_client_v(rd32_val); | ||
762 | mmfault->replayable_fault = | ||
763 | gmmu_fault_buf_entry_replayable_fault_v(rd32_val); | ||
764 | |||
765 | mmfault->fault_type = | ||
766 | gmmu_fault_buf_entry_fault_type_v(rd32_val); | ||
767 | mmfault->access_type = | ||
768 | gmmu_fault_buf_entry_access_type_v(rd32_val); | ||
769 | |||
770 | mmfault->client_type = | ||
771 | gmmu_fault_buf_entry_mmu_client_type_v(rd32_val); | ||
772 | |||
773 | mmfault->gpc_id = | ||
774 | gmmu_fault_buf_entry_gpc_id_v(rd32_val); | ||
775 | mmfault->protected_mode = | ||
776 | gmmu_fault_buf_entry_protected_mode_v(rd32_val); | ||
777 | |||
778 | mmfault->replay_fault_en = | ||
779 | gmmu_fault_buf_entry_replayable_fault_en_v(rd32_val); | ||
780 | |||
781 | mmfault->valid = gmmu_fault_buf_entry_valid_v(rd32_val); | ||
782 | |||
783 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
784 | gmmu_fault_buf_entry_fault_type_w()); | ||
785 | rd32_val &= ~(gmmu_fault_buf_entry_valid_m()); | ||
786 | nvgpu_mem_wr32(g, mem, offset + gmmu_fault_buf_entry_valid_w(), | ||
787 | rd32_val); | ||
788 | |||
789 | gv11b_fb_parse_mmfault(mmfault); | ||
790 | } | ||
791 | |||
792 | static void gv11b_fb_handle_mmu_fault_common(struct gk20a *g, | ||
793 | struct mmu_fault_info *mmfault, u32 *invalidate_replay_val) | ||
794 | { | ||
795 | unsigned int id_type; | ||
796 | u32 num_lce, act_eng_bitmask = 0; | ||
797 | int err = 0; | ||
798 | u32 id = ((u32)~0); | ||
799 | |||
800 | if (!mmfault->valid) | ||
801 | return; | ||
802 | |||
803 | gv11b_fb_print_fault_info(g, mmfault); | ||
804 | |||
805 | num_lce = gv11b_ce_get_num_lce(g); | ||
806 | if ((mmfault->mmu_engine_id >= | ||
807 | gmmu_fault_mmu_eng_id_ce0_v()) && | ||
808 | (mmfault->mmu_engine_id < | ||
809 | gmmu_fault_mmu_eng_id_ce0_v() + num_lce)) { | ||
810 | /* CE page faults are not reported as replayable */ | ||
811 | nvgpu_log(g, gpu_dbg_intr, "CE Faulted"); | ||
812 | err = gv11b_fb_fix_page_fault(g, mmfault); | ||
813 | gv11b_fifo_reset_pbdma_and_eng_faulted(g, mmfault->refch, | ||
814 | mmfault->faulted_pbdma, mmfault->faulted_engine); | ||
815 | if (!err) { | ||
816 | nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Fixed"); | ||
817 | *invalidate_replay_val = 0; | ||
818 | /* refch in mmfault is assigned at the time of copying | ||
819 | * fault info from snap reg or bar2 fault buf | ||
820 | */ | ||
821 | gk20a_channel_put(mmfault->refch); | ||
822 | return; | ||
823 | } | ||
824 | /* Do recovery. Channel recovery needs refch */ | ||
825 | nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Not Fixed"); | ||
826 | } | ||
827 | |||
828 | if (!mmfault->replayable_fault) { | ||
829 | if (mmfault->fault_type == | ||
830 | gmmu_fault_type_unbound_inst_block_v()) { | ||
831 | /* | ||
832 | * Bug 1847172: When an engine faults due to an unbound | ||
833 | * instance block, the fault cannot be isolated to a | ||
834 | * single context so we need to reset the entire runlist | ||
835 | */ | ||
836 | id_type = ID_TYPE_UNKNOWN; | ||
837 | |||
838 | } else if (mmfault->refch) { | ||
839 | if (gk20a_is_channel_marked_as_tsg(mmfault->refch)) { | ||
840 | id = mmfault->refch->tsgid; | ||
841 | id_type = ID_TYPE_TSG; | ||
842 | } else { | ||
843 | id = mmfault->chid; | ||
844 | id_type = ID_TYPE_CHANNEL; | ||
845 | } | ||
846 | if (mmfault->refch->mmu_nack_handled) { | ||
847 | /* We have already recovered for the same | ||
848 | * context, skip doing another recovery. | ||
849 | */ | ||
850 | mmfault->refch->mmu_nack_handled = false; | ||
851 | /* | ||
852 | * Recovery path can be entered twice for the | ||
853 | * same error in case of mmu nack. If mmu | ||
854 | * nack interrupt is handled before mmu fault | ||
855 | * then channel reference is increased to avoid | ||
856 | * closing the channel by userspace. Decrement | ||
857 | * channel reference. | ||
858 | */ | ||
859 | gk20a_channel_put(mmfault->refch); | ||
860 | /* refch in mmfault is assigned at the time | ||
861 | * of copying fault info from snap reg or bar2 | ||
862 | * fault buf. | ||
863 | */ | ||
864 | gk20a_channel_put(mmfault->refch); | ||
865 | return; | ||
866 | } | ||
867 | } else { | ||
868 | id_type = ID_TYPE_UNKNOWN; | ||
869 | } | ||
870 | if (mmfault->faulted_engine != FIFO_INVAL_ENGINE_ID) | ||
871 | act_eng_bitmask = BIT(mmfault->faulted_engine); | ||
872 | |||
873 | /* Indicate recovery is handled if mmu fault is a result of | ||
874 | * mmu nack. | ||
875 | */ | ||
876 | mmfault->refch->mmu_nack_handled = true; | ||
877 | g->ops.fifo.teardown_ch_tsg(g, act_eng_bitmask, | ||
878 | id, id_type, RC_TYPE_MMU_FAULT, mmfault); | ||
879 | } else { | ||
880 | if (mmfault->fault_type == gmmu_fault_type_pte_v()) { | ||
881 | nvgpu_log(g, gpu_dbg_intr, "invalid pte! try to fix"); | ||
882 | err = gv11b_fb_fix_page_fault(g, mmfault); | ||
883 | if (err) | ||
884 | *invalidate_replay_val |= | ||
885 | fb_mmu_invalidate_replay_cancel_global_f(); | ||
886 | else | ||
887 | *invalidate_replay_val |= | ||
888 | fb_mmu_invalidate_replay_start_ack_all_f(); | ||
889 | } else { | ||
890 | /* cancel faults other than invalid pte */ | ||
891 | *invalidate_replay_val |= | ||
892 | fb_mmu_invalidate_replay_cancel_global_f(); | ||
893 | } | ||
894 | /* refch in mmfault is assigned at the time of copying | ||
895 | * fault info from snap reg or bar2 fault buf | ||
896 | */ | ||
897 | gk20a_channel_put(mmfault->refch); | ||
898 | } | ||
899 | } | ||
900 | |||
901 | static int gv11b_fb_replay_or_cancel_faults(struct gk20a *g, | ||
902 | u32 invalidate_replay_val) | ||
903 | { | ||
904 | int err = 0; | ||
905 | |||
906 | nvgpu_log_fn(g, " "); | ||
907 | |||
908 | if (invalidate_replay_val & | ||
909 | fb_mmu_invalidate_replay_cancel_global_f()) { | ||
910 | /* | ||
911 | * cancel faults so that next time it faults as | ||
912 | * replayable faults and channel recovery can be done | ||
913 | */ | ||
914 | err = g->ops.fb.mmu_invalidate_replay(g, | ||
915 | fb_mmu_invalidate_replay_cancel_global_f()); | ||
916 | } else if (invalidate_replay_val & | ||
917 | fb_mmu_invalidate_replay_start_ack_all_f()) { | ||
918 | /* pte valid is fixed. replay faulting request */ | ||
919 | err = g->ops.fb.mmu_invalidate_replay(g, | ||
920 | fb_mmu_invalidate_replay_start_ack_all_f()); | ||
921 | } | ||
922 | |||
923 | return err; | ||
924 | } | ||
925 | |||
926 | void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g, | ||
927 | u32 fault_status, u32 index) | ||
928 | { | ||
929 | u32 get_indx, offset, rd32_val, entries; | ||
930 | struct nvgpu_mem *mem; | ||
931 | struct mmu_fault_info *mmfault; | ||
932 | u32 invalidate_replay_val = 0; | ||
933 | u64 prev_fault_addr = 0ULL; | ||
934 | u64 next_fault_addr = 0ULL; | ||
935 | |||
936 | if (gv11b_fb_is_fault_buffer_empty(g, index, &get_indx)) { | ||
937 | nvgpu_log(g, gpu_dbg_intr, | ||
938 | "SPURIOUS mmu fault: reg index:%d", index); | ||
939 | return; | ||
940 | } | ||
941 | nvgpu_log(g, gpu_dbg_intr, "%s MMU FAULT" , | ||
942 | index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX ? | ||
943 | "REPLAY" : "NON-REPLAY"); | ||
944 | |||
945 | nvgpu_log(g, gpu_dbg_intr, "get ptr = %d", get_indx); | ||
946 | |||
947 | mem = &g->mm.hw_fault_buf[index]; | ||
948 | mmfault = &g->mm.fault_info[index]; | ||
949 | |||
950 | entries = gv11b_fb_fault_buffer_size_val(g, index); | ||
951 | nvgpu_log(g, gpu_dbg_intr, "buffer num entries = %d", entries); | ||
952 | |||
953 | offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); | ||
954 | nvgpu_log(g, gpu_dbg_intr, "starting word offset = 0x%x", offset); | ||
955 | |||
956 | rd32_val = nvgpu_mem_rd32(g, mem, | ||
957 | offset + gmmu_fault_buf_entry_valid_w()); | ||
958 | nvgpu_log(g, gpu_dbg_intr, "entry valid offset val = 0x%x", rd32_val); | ||
959 | |||
960 | while ((rd32_val & gmmu_fault_buf_entry_valid_m())) { | ||
961 | |||
962 | nvgpu_log(g, gpu_dbg_intr, "entry valid = 0x%x", rd32_val); | ||
963 | |||
964 | gv11b_fb_copy_from_hw_fault_buf(g, mem, offset, mmfault); | ||
965 | |||
966 | get_indx = (get_indx + 1) % entries; | ||
967 | nvgpu_log(g, gpu_dbg_intr, "new get index = %d", get_indx); | ||
968 | |||
969 | gv11b_fb_fault_buffer_get_ptr_update(g, index, get_indx); | ||
970 | |||
971 | offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); | ||
972 | nvgpu_log(g, gpu_dbg_intr, "next word offset = 0x%x", offset); | ||
973 | |||
974 | rd32_val = nvgpu_mem_rd32(g, mem, | ||
975 | offset + gmmu_fault_buf_entry_valid_w()); | ||
976 | |||
977 | if (index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX && | ||
978 | mmfault->fault_addr != 0ULL) { | ||
979 | /* fault_addr "0" is not supposed to be fixed ever. | ||
980 | * For the first time when prev = 0, next = 0 and | ||
981 | * fault addr is also 0 then handle_mmu_fault_common will | ||
982 | * not be called. Fix by checking fault_addr not equal to 0 | ||
983 | */ | ||
984 | prev_fault_addr = next_fault_addr; | ||
985 | next_fault_addr = mmfault->fault_addr; | ||
986 | if (prev_fault_addr == next_fault_addr) { | ||
987 | nvgpu_log(g, gpu_dbg_intr, "pte already scanned"); | ||
988 | if (mmfault->refch) | ||
989 | gk20a_channel_put(mmfault->refch); | ||
990 | continue; | ||
991 | } | ||
992 | } | ||
993 | |||
994 | gv11b_fb_handle_mmu_fault_common(g, mmfault, | ||
995 | &invalidate_replay_val); | ||
996 | |||
997 | } | ||
998 | if (index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX && | ||
999 | invalidate_replay_val != 0U) | ||
1000 | gv11b_fb_replay_or_cancel_faults(g, invalidate_replay_val); | ||
1001 | } | ||
1002 | |||
1003 | static void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g, | ||
1004 | u32 fault_status, struct mmu_fault_info *mmfault) | ||
1005 | { | ||
1006 | u32 reg_val; | ||
1007 | u32 addr_lo, addr_hi; | ||
1008 | u64 inst_ptr; | ||
1009 | int chid = FIFO_INVAL_CHANNEL_ID; | ||
1010 | struct channel_gk20a *refch; | ||
1011 | |||
1012 | memset(mmfault, 0, sizeof(*mmfault)); | ||
1013 | |||
1014 | if (!(fault_status & fb_mmu_fault_status_valid_set_f())) { | ||
1015 | |||
1016 | nvgpu_log(g, gpu_dbg_intr, "mmu fault status valid not set"); | ||
1017 | return; | ||
1018 | } | ||
1019 | |||
1020 | g->ops.fb.read_mmu_fault_inst_lo_hi(g, ®_val, &addr_hi); | ||
1021 | |||
1022 | addr_lo = fb_mmu_fault_inst_lo_addr_v(reg_val); | ||
1023 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
1024 | |||
1025 | addr_hi = fb_mmu_fault_inst_hi_addr_v(addr_hi); | ||
1026 | inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
1027 | |||
1028 | /* refch will be put back after fault is handled */ | ||
1029 | refch = gk20a_refch_from_inst_ptr(g, inst_ptr); | ||
1030 | if (refch) | ||
1031 | chid = refch->chid; | ||
1032 | |||
1033 | /* It is still ok to continue if refch is NULL */ | ||
1034 | mmfault->refch = refch; | ||
1035 | mmfault->chid = chid; | ||
1036 | mmfault->inst_ptr = inst_ptr; | ||
1037 | mmfault->inst_aperture = fb_mmu_fault_inst_lo_aperture_v(reg_val); | ||
1038 | mmfault->mmu_engine_id = fb_mmu_fault_inst_lo_engine_id_v(reg_val); | ||
1039 | |||
1040 | gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, | ||
1041 | &mmfault->faulted_engine, &mmfault->faulted_subid, | ||
1042 | &mmfault->faulted_pbdma); | ||
1043 | |||
1044 | g->ops.fb.read_mmu_fault_addr_lo_hi(g, ®_val, &addr_hi); | ||
1045 | |||
1046 | addr_lo = fb_mmu_fault_addr_lo_addr_v(reg_val); | ||
1047 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
1048 | |||
1049 | mmfault->fault_addr_aperture = | ||
1050 | fb_mmu_fault_addr_lo_phys_aperture_v(reg_val); | ||
1051 | |||
1052 | addr_hi = fb_mmu_fault_addr_hi_addr_v(addr_hi); | ||
1053 | mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
1054 | |||
1055 | reg_val = g->ops.fb.read_mmu_fault_info(g); | ||
1056 | mmfault->fault_type = fb_mmu_fault_info_fault_type_v(reg_val); | ||
1057 | mmfault->replayable_fault = | ||
1058 | fb_mmu_fault_info_replayable_fault_v(reg_val); | ||
1059 | mmfault->client_id = fb_mmu_fault_info_client_v(reg_val); | ||
1060 | mmfault->access_type = fb_mmu_fault_info_access_type_v(reg_val); | ||
1061 | mmfault->client_type = fb_mmu_fault_info_client_type_v(reg_val); | ||
1062 | mmfault->gpc_id = fb_mmu_fault_info_gpc_id_v(reg_val); | ||
1063 | mmfault->protected_mode = | ||
1064 | fb_mmu_fault_info_protected_mode_v(reg_val); | ||
1065 | mmfault->replay_fault_en = | ||
1066 | fb_mmu_fault_info_replayable_fault_en_v(reg_val); | ||
1067 | |||
1068 | mmfault->valid = fb_mmu_fault_info_valid_v(reg_val); | ||
1069 | |||
1070 | fault_status &= ~(fb_mmu_fault_status_valid_m()); | ||
1071 | g->ops.fb.write_mmu_fault_status(g, fault_status); | ||
1072 | |||
1073 | gv11b_fb_parse_mmfault(mmfault); | ||
1074 | |||
1075 | } | ||
1076 | |||
1077 | void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g, | ||
1078 | u32 fault_status) | ||
1079 | { | ||
1080 | u32 reg_val; | ||
1081 | u32 index = NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX; | ||
1082 | |||
1083 | reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); | ||
1084 | |||
1085 | if (fault_status & | ||
1086 | fb_mmu_fault_status_replayable_getptr_corrupted_m()) { | ||
1087 | |||
1088 | nvgpu_err(g, "replayable getptr corrupted set"); | ||
1089 | |||
1090 | gv11b_fb_fault_buf_configure_hw(g, index); | ||
1091 | |||
1092 | reg_val = set_field(reg_val, | ||
1093 | fb_mmu_fault_buffer_get_getptr_corrupted_m(), | ||
1094 | fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); | ||
1095 | } | ||
1096 | |||
1097 | if (fault_status & | ||
1098 | fb_mmu_fault_status_replayable_overflow_m()) { | ||
1099 | bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); | ||
1100 | |||
1101 | nvgpu_err(g, "replayable overflow: buffer full:%s", | ||
1102 | buffer_full?"true":"false"); | ||
1103 | |||
1104 | reg_val = set_field(reg_val, | ||
1105 | fb_mmu_fault_buffer_get_overflow_m(), | ||
1106 | fb_mmu_fault_buffer_get_overflow_clear_f()); | ||
1107 | } | ||
1108 | |||
1109 | g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); | ||
1110 | } | ||
1111 | |||
1112 | void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g, | ||
1113 | u32 fault_status) | ||
1114 | { | ||
1115 | u32 reg_val; | ||
1116 | u32 index = NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX; | ||
1117 | |||
1118 | reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); | ||
1119 | |||
1120 | if (fault_status & | ||
1121 | fb_mmu_fault_status_non_replayable_getptr_corrupted_m()) { | ||
1122 | |||
1123 | nvgpu_err(g, "non replayable getptr corrupted set"); | ||
1124 | |||
1125 | gv11b_fb_fault_buf_configure_hw(g, index); | ||
1126 | |||
1127 | reg_val = set_field(reg_val, | ||
1128 | fb_mmu_fault_buffer_get_getptr_corrupted_m(), | ||
1129 | fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); | ||
1130 | } | ||
1131 | |||
1132 | if (fault_status & | ||
1133 | fb_mmu_fault_status_non_replayable_overflow_m()) { | ||
1134 | |||
1135 | bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); | ||
1136 | |||
1137 | nvgpu_err(g, "non replayable overflow: buffer full:%s", | ||
1138 | buffer_full?"true":"false"); | ||
1139 | |||
1140 | reg_val = set_field(reg_val, | ||
1141 | fb_mmu_fault_buffer_get_overflow_m(), | ||
1142 | fb_mmu_fault_buffer_get_overflow_clear_f()); | ||
1143 | } | ||
1144 | |||
1145 | g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); | ||
1146 | } | ||
1147 | |||
1148 | static void gv11b_fb_handle_bar2_fault(struct gk20a *g, | ||
1149 | struct mmu_fault_info *mmfault, u32 fault_status) | ||
1150 | { | ||
1151 | if (fault_status & fb_mmu_fault_status_non_replayable_error_m()) { | ||
1152 | if (gv11b_fb_is_fault_buf_enabled(g, | ||
1153 | NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) | ||
1154 | gv11b_fb_fault_buf_configure_hw(g, NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX); | ||
1155 | } | ||
1156 | |||
1157 | if (fault_status & fb_mmu_fault_status_replayable_error_m()) { | ||
1158 | if (gv11b_fb_is_fault_buf_enabled(g, | ||
1159 | NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX)) | ||
1160 | gv11b_fb_fault_buf_configure_hw(g, | ||
1161 | NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); | ||
1162 | } | ||
1163 | gv11b_ce_mthd_buffer_fault_in_bar2_fault(g); | ||
1164 | |||
1165 | g->ops.bus.bar2_bind(g, &g->mm.bar2.inst_block); | ||
1166 | |||
1167 | if (mmfault->refch) { | ||
1168 | gk20a_channel_put(mmfault->refch); | ||
1169 | mmfault->refch = NULL; | ||
1170 | } | ||
1171 | } | ||
1172 | |||
1173 | void gv11b_fb_handle_other_fault_notify(struct gk20a *g, | ||
1174 | u32 fault_status) | ||
1175 | { | ||
1176 | struct mmu_fault_info *mmfault; | ||
1177 | u32 invalidate_replay_val = 0; | ||
1178 | |||
1179 | mmfault = &g->mm.fault_info[NVGPU_MM_MMU_FAULT_TYPE_OTHER_AND_NONREPLAY]; | ||
1180 | |||
1181 | gv11b_mm_copy_from_fault_snap_reg(g, fault_status, mmfault); | ||
1182 | |||
1183 | /* BAR2/Physical faults will not be snapped in hw fault buf */ | ||
1184 | if (mmfault->mmu_engine_id == gmmu_fault_mmu_eng_id_bar2_v()) { | ||
1185 | nvgpu_err(g, "BAR2 MMU FAULT"); | ||
1186 | gv11b_fb_handle_bar2_fault(g, mmfault, fault_status); | ||
1187 | |||
1188 | } else if (mmfault->mmu_engine_id == | ||
1189 | gmmu_fault_mmu_eng_id_physical_v()) { | ||
1190 | /* usually means VPR or out of bounds physical accesses */ | ||
1191 | nvgpu_err(g, "PHYSICAL MMU FAULT"); | ||
1192 | |||
1193 | } else { | ||
1194 | gv11b_fb_handle_mmu_fault_common(g, mmfault, | ||
1195 | &invalidate_replay_val); | ||
1196 | |||
1197 | if (invalidate_replay_val) | ||
1198 | gv11b_fb_replay_or_cancel_faults(g, | ||
1199 | invalidate_replay_val); | ||
1200 | } | ||
1201 | } | ||
1202 | |||
1203 | void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status) | ||
1204 | { | ||
1205 | u32 dropped_faults = 0; | ||
1206 | |||
1207 | dropped_faults = fb_mmu_fault_status_dropped_bar1_phys_set_f() | | ||
1208 | fb_mmu_fault_status_dropped_bar1_virt_set_f() | | ||
1209 | fb_mmu_fault_status_dropped_bar2_phys_set_f() | | ||
1210 | fb_mmu_fault_status_dropped_bar2_virt_set_f() | | ||
1211 | fb_mmu_fault_status_dropped_ifb_phys_set_f() | | ||
1212 | fb_mmu_fault_status_dropped_ifb_virt_set_f() | | ||
1213 | fb_mmu_fault_status_dropped_other_phys_set_f()| | ||
1214 | fb_mmu_fault_status_dropped_other_virt_set_f(); | ||
1215 | |||
1216 | if (fault_status & dropped_faults) { | ||
1217 | nvgpu_err(g, "dropped mmu fault (0x%08x)", | ||
1218 | fault_status & dropped_faults); | ||
1219 | g->ops.fb.write_mmu_fault_status(g, dropped_faults); | ||
1220 | } | ||
1221 | } | ||
1222 | |||
1223 | void gv11b_fb_handle_replayable_mmu_fault(struct gk20a *g) | ||
1224 | { | ||
1225 | u32 fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); | ||
1226 | |||
1227 | if (!(fault_status & fb_mmu_fault_status_replayable_m())) | ||
1228 | return; | ||
1229 | |||
1230 | if (gv11b_fb_is_fault_buf_enabled(g, | ||
1231 | NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) { | ||
1232 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1233 | fault_status, | ||
1234 | NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); | ||
1235 | } | ||
1236 | } | ||
1237 | |||
1238 | static void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr) | ||
1239 | { | ||
1240 | u32 fault_status = g->ops.fb.read_mmu_fault_status(g); | ||
1241 | |||
1242 | nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status); | ||
1243 | |||
1244 | if (niso_intr & | ||
1245 | fb_niso_intr_mmu_other_fault_notify_m()) { | ||
1246 | |||
1247 | gv11b_fb_handle_dropped_mmu_fault(g, fault_status); | ||
1248 | |||
1249 | gv11b_fb_handle_other_fault_notify(g, fault_status); | ||
1250 | } | ||
1251 | |||
1252 | if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) { | ||
1253 | |||
1254 | if (niso_intr & | ||
1255 | fb_niso_intr_mmu_nonreplayable_fault_notify_m()) { | ||
1256 | |||
1257 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1258 | fault_status, | ||
1259 | NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX); | ||
1260 | |||
1261 | /* | ||
1262 | * When all the faults are processed, | ||
1263 | * GET and PUT will have same value and mmu fault status | ||
1264 | * bit will be reset by HW | ||
1265 | */ | ||
1266 | } | ||
1267 | if (niso_intr & | ||
1268 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m()) { | ||
1269 | |||
1270 | gv11b_fb_handle_nonreplay_fault_overflow(g, | ||
1271 | fault_status); | ||
1272 | } | ||
1273 | |||
1274 | } | ||
1275 | |||
1276 | if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX)) { | ||
1277 | |||
1278 | if (niso_intr & | ||
1279 | fb_niso_intr_mmu_replayable_fault_notify_m()) { | ||
1280 | |||
1281 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1282 | fault_status, | ||
1283 | NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); | ||
1284 | } | ||
1285 | if (niso_intr & | ||
1286 | fb_niso_intr_mmu_replayable_fault_overflow_m()) { | ||
1287 | |||
1288 | gv11b_fb_handle_replay_fault_overflow(g, | ||
1289 | fault_status); | ||
1290 | } | ||
1291 | |||
1292 | } | ||
1293 | |||
1294 | nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status"); | ||
1295 | g->ops.fb.write_mmu_fault_status(g, | ||
1296 | fb_mmu_fault_status_valid_clear_f()); | ||
1297 | } | ||
1298 | |||
1299 | void gv11b_fb_hub_isr(struct gk20a *g) | ||
1300 | { | ||
1301 | u32 status, niso_intr; | ||
1302 | |||
1303 | nvgpu_mutex_acquire(&g->mm.hub_isr_mutex); | ||
1304 | |||
1305 | niso_intr = gk20a_readl(g, fb_niso_intr_r()); | ||
1306 | |||
1307 | nvgpu_log(g, gpu_dbg_intr, "enter hub isr, niso_intr = 0x%08x", | ||
1308 | niso_intr); | ||
1309 | |||
1310 | if (niso_intr & | ||
1311 | (fb_niso_intr_hub_access_counter_notify_m() | | ||
1312 | fb_niso_intr_hub_access_counter_error_m())) { | ||
1313 | |||
1314 | nvgpu_info(g, "hub access counter notify/error"); | ||
1315 | } | ||
1316 | if (niso_intr & | ||
1317 | fb_niso_intr_mmu_ecc_uncorrected_error_notify_pending_f()) { | ||
1318 | |||
1319 | nvgpu_info(g, "ecc uncorrected error notify"); | ||
1320 | |||
1321 | status = gk20a_readl(g, fb_mmu_l2tlb_ecc_status_r()); | ||
1322 | if (status) | ||
1323 | gv11b_handle_l2tlb_ecc_isr(g, status); | ||
1324 | |||
1325 | status = gk20a_readl(g, fb_mmu_hubtlb_ecc_status_r()); | ||
1326 | if (status) | ||
1327 | gv11b_handle_hubtlb_ecc_isr(g, status); | ||
1328 | |||
1329 | status = gk20a_readl(g, fb_mmu_fillunit_ecc_status_r()); | ||
1330 | if (status) | ||
1331 | gv11b_handle_fillunit_ecc_isr(g, status); | ||
1332 | } | ||
1333 | if (niso_intr & | ||
1334 | (fb_niso_intr_mmu_other_fault_notify_m() | | ||
1335 | fb_niso_intr_mmu_replayable_fault_notify_m() | | ||
1336 | fb_niso_intr_mmu_replayable_fault_overflow_m() | | ||
1337 | fb_niso_intr_mmu_nonreplayable_fault_notify_m() | | ||
1338 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) { | ||
1339 | |||
1340 | nvgpu_log(g, gpu_dbg_intr, "MMU Fault"); | ||
1341 | gv11b_fb_handle_mmu_fault(g, niso_intr); | ||
1342 | } | ||
1343 | |||
1344 | nvgpu_mutex_release(&g->mm.hub_isr_mutex); | ||
1345 | } | ||
1346 | |||
1347 | bool gv11b_fb_mmu_fault_pending(struct gk20a *g) | ||
1348 | { | ||
1349 | if (gk20a_readl(g, fb_niso_intr_r()) & | ||
1350 | (fb_niso_intr_mmu_other_fault_notify_m() | | ||
1351 | fb_niso_intr_mmu_ecc_uncorrected_error_notify_m() | | ||
1352 | fb_niso_intr_mmu_replayable_fault_notify_m() | | ||
1353 | fb_niso_intr_mmu_replayable_fault_overflow_m() | | ||
1354 | fb_niso_intr_mmu_nonreplayable_fault_notify_m() | | ||
1355 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) | ||
1356 | return true; | ||
1357 | |||
1358 | return false; | ||
1359 | } | ||
1360 | |||
1361 | int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, | ||
1362 | u32 invalidate_replay_val) | ||
1363 | { | ||
1364 | int err = -ETIMEDOUT; | ||
1365 | u32 reg_val; | ||
1366 | struct nvgpu_timeout timeout; | ||
1367 | |||
1368 | nvgpu_log_fn(g, " "); | ||
1369 | |||
1370 | nvgpu_mutex_acquire(&g->mm.tlb_lock); | ||
1371 | |||
1372 | reg_val = gk20a_readl(g, fb_mmu_invalidate_r()); | ||
1373 | |||
1374 | reg_val |= fb_mmu_invalidate_all_va_true_f() | | ||
1375 | fb_mmu_invalidate_all_pdb_true_f() | | ||
1376 | invalidate_replay_val | | ||
1377 | fb_mmu_invalidate_trigger_true_f(); | ||
1378 | |||
1379 | gk20a_writel(g, fb_mmu_invalidate_r(), reg_val); | ||
1380 | |||
1381 | /* retry 200 times */ | ||
1382 | nvgpu_timeout_init(g, &timeout, 200, NVGPU_TIMER_RETRY_TIMER); | ||
1383 | do { | ||
1384 | reg_val = gk20a_readl(g, fb_mmu_ctrl_r()); | ||
1385 | if (fb_mmu_ctrl_pri_fifo_empty_v(reg_val) != | ||
1386 | fb_mmu_ctrl_pri_fifo_empty_false_f()) { | ||
1387 | err = 0; | ||
1388 | break; | ||
1389 | } | ||
1390 | nvgpu_udelay(5); | ||
1391 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
1392 | "invalidate replay failed on 0x%llx")); | ||
1393 | if (err) | ||
1394 | nvgpu_err(g, "invalidate replay timedout"); | ||
1395 | |||
1396 | nvgpu_mutex_release(&g->mm.tlb_lock); | ||
1397 | |||
1398 | return err; | ||
1399 | } | ||
1400 | |||
1401 | static int gv11b_fb_fix_page_fault(struct gk20a *g, | ||
1402 | struct mmu_fault_info *mmfault) | ||
1403 | { | ||
1404 | int err = 0; | ||
1405 | u32 pte[2]; | ||
1406 | |||
1407 | if (mmfault->refch == NULL) { | ||
1408 | nvgpu_log(g, gpu_dbg_intr, "refch from mmu_fault_info is NULL"); | ||
1409 | return -EINVAL; | ||
1410 | } | ||
1411 | |||
1412 | err = __nvgpu_get_pte(g, | ||
1413 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1414 | if (err) { | ||
1415 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not found"); | ||
1416 | return err; | ||
1417 | } | ||
1418 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1419 | "pte: %#08x %#08x", pte[1], pte[0]); | ||
1420 | |||
1421 | if (pte[0] == 0x0 && pte[1] == 0x0) { | ||
1422 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1423 | "pte all zeros, do not set valid"); | ||
1424 | return -1; | ||
1425 | } | ||
1426 | if (pte[0] & gmmu_new_pte_valid_true_f()) { | ||
1427 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1428 | "pte valid already set"); | ||
1429 | return -1; | ||
1430 | } | ||
1431 | |||
1432 | pte[0] |= gmmu_new_pte_valid_true_f(); | ||
1433 | if (pte[0] & gmmu_new_pte_read_only_true_f()) | ||
1434 | pte[0] &= ~(gmmu_new_pte_read_only_true_f()); | ||
1435 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1436 | "new pte: %#08x %#08x", pte[1], pte[0]); | ||
1437 | |||
1438 | err = __nvgpu_set_pte(g, | ||
1439 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1440 | if (err) { | ||
1441 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not fixed"); | ||
1442 | return err; | ||
1443 | } | ||
1444 | /* invalidate tlb so that GMMU does not use old cached translation */ | ||
1445 | g->ops.fb.tlb_invalidate(g, mmfault->refch->vm->pdb.mem); | ||
1446 | |||
1447 | err = __nvgpu_get_pte(g, | ||
1448 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1449 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1450 | "pte after tlb invalidate: %#08x %#08x", | ||
1451 | pte[1], pte[0]); | ||
1452 | return err; | ||
1453 | } | ||
1454 | |||
1455 | void fb_gv11b_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index, | ||
1456 | u32 addr_lo, u32 addr_hi) | ||
1457 | { | ||
1458 | nvgpu_writel(g, fb_mmu_fault_buffer_lo_r(index), addr_lo); | ||
1459 | nvgpu_writel(g, fb_mmu_fault_buffer_hi_r(index), addr_hi); | ||
1460 | } | ||
1461 | |||
1462 | u32 fb_gv11b_read_mmu_fault_buffer_get(struct gk20a *g, u32 index) | ||
1463 | { | ||
1464 | return nvgpu_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
1465 | } | ||
1466 | |||
1467 | void fb_gv11b_write_mmu_fault_buffer_get(struct gk20a *g, u32 index, | ||
1468 | u32 reg_val) | ||
1469 | { | ||
1470 | nvgpu_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); | ||
1471 | } | ||
1472 | |||
1473 | u32 fb_gv11b_read_mmu_fault_buffer_put(struct gk20a *g, u32 index) | ||
1474 | { | ||
1475 | return nvgpu_readl(g, fb_mmu_fault_buffer_put_r(index)); | ||
1476 | } | ||
1477 | |||
1478 | u32 fb_gv11b_read_mmu_fault_buffer_size(struct gk20a *g, u32 index) | ||
1479 | { | ||
1480 | return nvgpu_readl(g, fb_mmu_fault_buffer_size_r(index)); | ||
1481 | } | ||
1482 | |||
1483 | void fb_gv11b_write_mmu_fault_buffer_size(struct gk20a *g, u32 index, | ||
1484 | u32 reg_val) | ||
1485 | { | ||
1486 | nvgpu_writel(g, fb_mmu_fault_buffer_size_r(index), reg_val); | ||
1487 | } | ||
1488 | |||
1489 | void fb_gv11b_read_mmu_fault_addr_lo_hi(struct gk20a *g, | ||
1490 | u32 *addr_lo, u32 *addr_hi) | ||
1491 | { | ||
1492 | *addr_lo = nvgpu_readl(g, fb_mmu_fault_addr_lo_r()); | ||
1493 | *addr_hi = nvgpu_readl(g, fb_mmu_fault_addr_hi_r()); | ||
1494 | } | ||
1495 | |||
1496 | void fb_gv11b_read_mmu_fault_inst_lo_hi(struct gk20a *g, | ||
1497 | u32 *inst_lo, u32 *inst_hi) | ||
1498 | { | ||
1499 | *inst_lo = nvgpu_readl(g, fb_mmu_fault_inst_lo_r()); | ||
1500 | *inst_hi = nvgpu_readl(g, fb_mmu_fault_inst_hi_r()); | ||
1501 | } | ||
1502 | |||
1503 | u32 fb_gv11b_read_mmu_fault_info(struct gk20a *g) | ||
1504 | { | ||
1505 | return nvgpu_readl(g, fb_mmu_fault_info_r()); | ||
1506 | } | ||
1507 | |||
1508 | u32 fb_gv11b_read_mmu_fault_status(struct gk20a *g) | ||
1509 | { | ||
1510 | return nvgpu_readl(g, fb_mmu_fault_status_r()); | ||
1511 | } | ||
1512 | |||
1513 | void fb_gv11b_write_mmu_fault_status(struct gk20a *g, u32 reg_val) | ||
1514 | { | ||
1515 | nvgpu_writel(g, fb_mmu_fault_status_r(), reg_val); | ||
1516 | } | ||
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gv11b.h b/drivers/gpu/nvgpu/common/fb/fb_gv11b.h new file mode 100644 index 00000000..71fb3a41 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gv11b.h | |||
@@ -0,0 +1,82 @@ | |||
1 | /* | ||
2 | * GV11B FB | ||
3 | * | ||
4 | * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef _NVGPU_GV11B_FB | ||
26 | #define _NVGPU_GV11B_FB | ||
27 | |||
28 | #define NONREPLAY_REG_INDEX 0 | ||
29 | #define REPLAY_REG_INDEX 1 | ||
30 | |||
31 | struct gk20a; | ||
32 | struct gr_gk20a; | ||
33 | |||
34 | void gv11b_fb_init_hw(struct gk20a *g); | ||
35 | |||
36 | void gv11b_fb_init_fs_state(struct gk20a *g); | ||
37 | void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr); | ||
38 | void gv11b_fb_reset(struct gk20a *g); | ||
39 | void gv11b_fb_hub_isr(struct gk20a *g); | ||
40 | |||
41 | bool gv11b_fb_is_fault_buf_enabled(struct gk20a *g, u32 index ); | ||
42 | void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, | ||
43 | u32 index, u32 state); | ||
44 | void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, u32 index); | ||
45 | void gv11b_fb_enable_hub_intr(struct gk20a *g); | ||
46 | void gv11b_fb_disable_hub_intr(struct gk20a *g); | ||
47 | bool gv11b_fb_mmu_fault_pending(struct gk20a *g); | ||
48 | void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status); | ||
49 | void gv11b_fb_handle_other_fault_notify(struct gk20a *g, | ||
50 | u32 fault_status); | ||
51 | void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g, | ||
52 | u32 fault_status, u32 index); | ||
53 | void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g, | ||
54 | u32 fault_status); | ||
55 | void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g, | ||
56 | u32 fault_status); | ||
57 | void gv11b_fb_handle_replayable_mmu_fault(struct gk20a *g); | ||
58 | void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status); | ||
59 | void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status); | ||
60 | void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status); | ||
61 | |||
62 | void fb_gv11b_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index, | ||
63 | u32 addr_lo, u32 addr_hi); | ||
64 | u32 fb_gv11b_read_mmu_fault_buffer_get(struct gk20a *g, u32 index); | ||
65 | void fb_gv11b_write_mmu_fault_buffer_get(struct gk20a *g, u32 index, | ||
66 | u32 reg_val); | ||
67 | u32 fb_gv11b_read_mmu_fault_buffer_put(struct gk20a *g, u32 index); | ||
68 | u32 fb_gv11b_read_mmu_fault_buffer_size(struct gk20a *g, u32 index); | ||
69 | void fb_gv11b_write_mmu_fault_buffer_size(struct gk20a *g, u32 index, | ||
70 | u32 reg_val); | ||
71 | void fb_gv11b_read_mmu_fault_addr_lo_hi(struct gk20a *g, | ||
72 | u32 *addr_lo, u32 *addr_hi); | ||
73 | void fb_gv11b_read_mmu_fault_inst_lo_hi(struct gk20a *g, | ||
74 | u32 *inst_lo, u32 *inst_hi); | ||
75 | u32 fb_gv11b_read_mmu_fault_info(struct gk20a *g); | ||
76 | u32 fb_gv11b_read_mmu_fault_status(struct gk20a *g); | ||
77 | void fb_gv11b_write_mmu_fault_status(struct gk20a *g, u32 reg_val); | ||
78 | |||
79 | int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, | ||
80 | u32 invalidate_replay_val); | ||
81 | |||
82 | #endif | ||