diff options
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gk20a.h | 1739 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/gk20a.h | 1758 |
2 files changed, 1765 insertions, 1732 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index dd63a105..0df88edf 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h | |||
@@ -1,4 +1,6 @@ | |||
1 | /* | 1 | /* |
2 | * This file is used as a temporary redirection header for <nvgpu/gk20a.h> | ||
3 | * | ||
2 | * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. | 4 | * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. |
3 | * | 5 | * |
4 | * GK20A Graphics | 6 | * GK20A Graphics |
@@ -21,1738 +23,11 @@ | |||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | 23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
22 | * DEALINGS IN THE SOFTWARE. | 24 | * DEALINGS IN THE SOFTWARE. |
23 | */ | 25 | */ |
24 | #ifndef GK20A_H | ||
25 | #define GK20A_H | ||
26 | |||
27 | struct gk20a; | ||
28 | struct fifo_gk20a; | ||
29 | struct channel_gk20a; | ||
30 | struct gr_gk20a; | ||
31 | struct sim_nvgpu; | ||
32 | struct gk20a_ctxsw_ucode_segments; | ||
33 | struct gk20a_fecs_trace; | ||
34 | struct gk20a_ctxsw_trace; | ||
35 | struct acr_desc; | ||
36 | struct nvgpu_mem_alloc_tracker; | ||
37 | struct dbg_profiler_object_data; | ||
38 | struct gk20a_debug_output; | ||
39 | struct nvgpu_clk_pll_debug_data; | ||
40 | struct nvgpu_nvhost_dev; | ||
41 | struct nvgpu_cpu_time_correlation_sample; | ||
42 | struct nvgpu_mem_sgt; | ||
43 | struct nvgpu_warpstate; | ||
44 | struct nvgpu_clk_session; | ||
45 | struct nvgpu_clk_arb; | ||
46 | #ifdef CONFIG_GK20A_CTXSW_TRACE | ||
47 | struct nvgpu_gpu_ctxsw_trace_filter; | ||
48 | #endif | ||
49 | struct priv_cmd_entry; | ||
50 | struct nvgpu_gpfifo_args; | ||
51 | |||
52 | #include <nvgpu/lock.h> | ||
53 | #include <nvgpu/thread.h> | ||
54 | |||
55 | #include <nvgpu/mm.h> | ||
56 | #include <nvgpu/as.h> | ||
57 | #include <nvgpu/log.h> | ||
58 | #include <nvgpu/pramin.h> | ||
59 | #include <nvgpu/acr/nvgpu_acr.h> | ||
60 | #include <nvgpu/kref.h> | ||
61 | #include <nvgpu/falcon.h> | ||
62 | #include <nvgpu/pmu.h> | ||
63 | #include <nvgpu/atomic.h> | ||
64 | #include <nvgpu/barrier.h> | ||
65 | #include <nvgpu/rwsem.h> | ||
66 | #include <nvgpu/nvlink.h> | ||
67 | #include <nvgpu/sim.h> | ||
68 | #include <nvgpu/ecc.h> | ||
69 | |||
70 | #include "clk_gk20a.h" | ||
71 | #include "ce2_gk20a.h" | ||
72 | #include "fifo_gk20a.h" | ||
73 | #include "tsg_gk20a.h" | ||
74 | #include "clk/clk.h" | ||
75 | #include "perf/perf.h" | ||
76 | #include "pmgr/pmgr.h" | ||
77 | #include "therm/thrm.h" | ||
78 | |||
79 | #ifdef CONFIG_DEBUG_FS | ||
80 | struct railgate_stats { | ||
81 | unsigned long last_rail_gate_start; | ||
82 | unsigned long last_rail_gate_complete; | ||
83 | unsigned long last_rail_ungate_start; | ||
84 | unsigned long last_rail_ungate_complete; | ||
85 | unsigned long total_rail_gate_time_ms; | ||
86 | unsigned long total_rail_ungate_time_ms; | ||
87 | unsigned long railgating_cycle_count; | ||
88 | }; | ||
89 | #endif | ||
90 | |||
91 | enum gk20a_cbc_op { | ||
92 | gk20a_cbc_op_clear, | ||
93 | gk20a_cbc_op_clean, | ||
94 | gk20a_cbc_op_invalidate, | ||
95 | }; | ||
96 | |||
97 | #define MC_INTR_UNIT_DISABLE false | ||
98 | #define MC_INTR_UNIT_ENABLE true | ||
99 | |||
100 | #define GPU_LIT_NUM_GPCS 0 | ||
101 | #define GPU_LIT_NUM_PES_PER_GPC 1 | ||
102 | #define GPU_LIT_NUM_ZCULL_BANKS 2 | ||
103 | #define GPU_LIT_NUM_TPC_PER_GPC 3 | ||
104 | #define GPU_LIT_NUM_SM_PER_TPC 4 | ||
105 | #define GPU_LIT_NUM_FBPS 5 | ||
106 | #define GPU_LIT_GPC_BASE 6 | ||
107 | #define GPU_LIT_GPC_STRIDE 7 | ||
108 | #define GPU_LIT_GPC_SHARED_BASE 8 | ||
109 | #define GPU_LIT_TPC_IN_GPC_BASE 9 | ||
110 | #define GPU_LIT_TPC_IN_GPC_STRIDE 10 | ||
111 | #define GPU_LIT_TPC_IN_GPC_SHARED_BASE 11 | ||
112 | #define GPU_LIT_PPC_IN_GPC_BASE 12 | ||
113 | #define GPU_LIT_PPC_IN_GPC_STRIDE 13 | ||
114 | #define GPU_LIT_PPC_IN_GPC_SHARED_BASE 14 | ||
115 | #define GPU_LIT_ROP_BASE 15 | ||
116 | #define GPU_LIT_ROP_STRIDE 16 | ||
117 | #define GPU_LIT_ROP_SHARED_BASE 17 | ||
118 | #define GPU_LIT_HOST_NUM_ENGINES 18 | ||
119 | #define GPU_LIT_HOST_NUM_PBDMA 19 | ||
120 | #define GPU_LIT_LTC_STRIDE 20 | ||
121 | #define GPU_LIT_LTS_STRIDE 21 | ||
122 | #define GPU_LIT_NUM_FBPAS 22 | ||
123 | #define GPU_LIT_FBPA_STRIDE 23 | ||
124 | #define GPU_LIT_FBPA_BASE 24 | ||
125 | #define GPU_LIT_FBPA_SHARED_BASE 25 | ||
126 | #define GPU_LIT_SM_PRI_STRIDE 26 | ||
127 | #define GPU_LIT_SMPC_PRI_BASE 27 | ||
128 | #define GPU_LIT_SMPC_PRI_SHARED_BASE 28 | ||
129 | #define GPU_LIT_SMPC_PRI_UNIQUE_BASE 29 | ||
130 | #define GPU_LIT_SMPC_PRI_STRIDE 30 | ||
131 | #define GPU_LIT_TWOD_CLASS 31 | ||
132 | #define GPU_LIT_THREED_CLASS 32 | ||
133 | #define GPU_LIT_COMPUTE_CLASS 33 | ||
134 | #define GPU_LIT_GPFIFO_CLASS 34 | ||
135 | #define GPU_LIT_I2M_CLASS 35 | ||
136 | #define GPU_LIT_DMA_COPY_CLASS 36 | ||
137 | #define GPU_LIT_GPC_PRIV_STRIDE 37 | ||
138 | |||
139 | #define nvgpu_get_litter_value(g, v) (g)->ops.get_litter_value((g), v) | ||
140 | |||
141 | enum nvgpu_unit; | ||
142 | |||
143 | enum nvgpu_flush_op; | ||
144 | enum gk20a_mem_rw_flag; | ||
145 | |||
146 | struct _resmgr_context; | ||
147 | struct nvgpu_gpfifo_entry; | ||
148 | |||
149 | struct nvgpu_gpfifo_userdata { | ||
150 | struct nvgpu_gpfifo_entry __user *entries; | ||
151 | struct _resmgr_context *context; | ||
152 | }; | ||
153 | |||
154 | /* | ||
155 | * gpu_ops should only contain function pointers! Non-function pointer members | ||
156 | * should go in struct gk20a or be implemented with the boolean flag API defined | ||
157 | * in nvgpu/enabled.h | ||
158 | */ | ||
159 | |||
160 | /* index for FB fault buffer functions */ | ||
161 | #define NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX 0U | ||
162 | #define NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX 1U | ||
163 | #define NVGPU_FB_MMU_FAULT_BUF_DISABLED 0U | ||
164 | #define NVGPU_FB_MMU_FAULT_BUF_ENABLED 1U | ||
165 | |||
166 | /* Parameters for init_elcg_mode/init_blcg_mode */ | ||
167 | enum { | ||
168 | ELCG_RUN, /* clk always run, i.e. disable elcg */ | ||
169 | ELCG_STOP, /* clk is stopped */ | ||
170 | ELCG_AUTO /* clk will run when non-idle, standard elcg mode */ | ||
171 | }; | ||
172 | |||
173 | enum { | ||
174 | BLCG_RUN, /* clk always run, i.e. disable blcg */ | ||
175 | BLCG_AUTO /* clk will run when non-idle, standard blcg mode */ | ||
176 | }; | ||
177 | |||
178 | struct gpu_ops { | ||
179 | struct { | ||
180 | int (*determine_L2_size_bytes)(struct gk20a *gk20a); | ||
181 | u64 (*get_cbc_base_divisor)(struct gk20a *g); | ||
182 | int (*init_comptags)(struct gk20a *g, struct gr_gk20a *gr); | ||
183 | int (*cbc_ctrl)(struct gk20a *g, enum gk20a_cbc_op op, | ||
184 | u32 min, u32 max); | ||
185 | void (*set_zbc_color_entry)(struct gk20a *g, | ||
186 | struct zbc_entry *color_val, | ||
187 | u32 index); | ||
188 | void (*set_zbc_depth_entry)(struct gk20a *g, | ||
189 | struct zbc_entry *depth_val, | ||
190 | u32 index); | ||
191 | void (*set_zbc_s_entry)(struct gk20a *g, | ||
192 | struct zbc_entry *s_val, | ||
193 | u32 index); | ||
194 | void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr); | ||
195 | void (*set_enabled)(struct gk20a *g, bool enabled); | ||
196 | void (*init_fs_state)(struct gk20a *g); | ||
197 | void (*isr)(struct gk20a *g); | ||
198 | u32 (*cbc_fix_config)(struct gk20a *g, int base); | ||
199 | void (*flush)(struct gk20a *g); | ||
200 | void (*intr_en_illegal_compstat)(struct gk20a *g, bool enable); | ||
201 | bool (*pri_is_ltc_addr)(struct gk20a *g, u32 addr); | ||
202 | bool (*is_ltcs_ltss_addr)(struct gk20a *g, u32 addr); | ||
203 | bool (*is_ltcn_ltss_addr)(struct gk20a *g, u32 addr); | ||
204 | void (*split_lts_broadcast_addr)(struct gk20a *g, u32 addr, | ||
205 | u32 *priv_addr_table, | ||
206 | u32 *priv_addr_table_index); | ||
207 | void (*split_ltc_broadcast_addr)(struct gk20a *g, u32 addr, | ||
208 | u32 *priv_addr_table, | ||
209 | u32 *priv_addr_table_index); | ||
210 | } ltc; | ||
211 | struct { | ||
212 | void (*isr_stall)(struct gk20a *g, u32 inst_id, u32 pri_base); | ||
213 | u32 (*isr_nonstall)(struct gk20a *g, u32 inst_id, u32 pri_base); | ||
214 | u32 (*get_num_pce)(struct gk20a *g); | ||
215 | } ce2; | ||
216 | struct { | ||
217 | u32 (*get_patch_slots)(struct gk20a *g); | ||
218 | int (*init_fs_state)(struct gk20a *g); | ||
219 | int (*init_preemption_state)(struct gk20a *g); | ||
220 | void (*access_smpc_reg)(struct gk20a *g, u32 quad, u32 offset); | ||
221 | void (*bundle_cb_defaults)(struct gk20a *g); | ||
222 | void (*cb_size_default)(struct gk20a *g); | ||
223 | int (*calc_global_ctx_buffer_size)(struct gk20a *g); | ||
224 | void (*commit_global_attrib_cb)(struct gk20a *g, | ||
225 | struct nvgpu_gr_ctx *ch_ctx, | ||
226 | u64 addr, bool patch); | ||
227 | void (*commit_global_bundle_cb)(struct gk20a *g, | ||
228 | struct nvgpu_gr_ctx *ch_ctx, | ||
229 | u64 addr, u64 size, bool patch); | ||
230 | int (*commit_global_cb_manager)(struct gk20a *g, | ||
231 | struct channel_gk20a *ch, | ||
232 | bool patch); | ||
233 | void (*commit_global_pagepool)(struct gk20a *g, | ||
234 | struct nvgpu_gr_ctx *ch_ctx, | ||
235 | u64 addr, u32 size, bool patch); | ||
236 | void (*init_gpc_mmu)(struct gk20a *g); | ||
237 | int (*handle_sw_method)(struct gk20a *g, u32 addr, | ||
238 | u32 class_num, u32 offset, u32 data); | ||
239 | void (*set_alpha_circular_buffer_size)(struct gk20a *g, | ||
240 | u32 data); | ||
241 | void (*set_circular_buffer_size)(struct gk20a *g, u32 data); | ||
242 | void (*set_bes_crop_debug3)(struct gk20a *g, u32 data); | ||
243 | void (*set_bes_crop_debug4)(struct gk20a *g, u32 data); | ||
244 | void (*enable_hww_exceptions)(struct gk20a *g); | ||
245 | bool (*is_valid_class)(struct gk20a *g, u32 class_num); | ||
246 | bool (*is_valid_gfx_class)(struct gk20a *g, u32 class_num); | ||
247 | bool (*is_valid_compute_class)(struct gk20a *g, u32 class_num); | ||
248 | void (*get_sm_dsm_perf_regs)(struct gk20a *g, | ||
249 | u32 *num_sm_dsm_perf_regs, | ||
250 | u32 **sm_dsm_perf_regs, | ||
251 | u32 *perf_register_stride); | ||
252 | void (*get_sm_dsm_perf_ctrl_regs)(struct gk20a *g, | ||
253 | u32 *num_sm_dsm_perf_regs, | ||
254 | u32 **sm_dsm_perf_regs, | ||
255 | u32 *perf_register_stride); | ||
256 | void (*get_ovr_perf_regs)(struct gk20a *g, | ||
257 | u32 *num_ovr_perf_regs, | ||
258 | u32 **ovr_perf_regsr); | ||
259 | void (*set_hww_esr_report_mask)(struct gk20a *g); | ||
260 | int (*setup_alpha_beta_tables)(struct gk20a *g, | ||
261 | struct gr_gk20a *gr); | ||
262 | int (*falcon_load_ucode)(struct gk20a *g, | ||
263 | u64 addr_base, | ||
264 | struct gk20a_ctxsw_ucode_segments *segments, | ||
265 | u32 reg_offset); | ||
266 | int (*load_ctxsw_ucode)(struct gk20a *g); | ||
267 | u32 (*get_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index); | ||
268 | void (*set_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index); | ||
269 | int (*alloc_obj_ctx)(struct channel_gk20a *c, | ||
270 | u32 class_num, u32 flags); | ||
271 | int (*bind_ctxsw_zcull)(struct gk20a *g, struct gr_gk20a *gr, | ||
272 | struct channel_gk20a *c, u64 zcull_va, | ||
273 | u32 mode); | ||
274 | int (*get_zcull_info)(struct gk20a *g, struct gr_gk20a *gr, | ||
275 | struct gr_zcull_info *zcull_params); | ||
276 | int (*decode_egpc_addr)(struct gk20a *g, | ||
277 | u32 addr, enum ctxsw_addr_type *addr_type, | ||
278 | u32 *gpc_num, u32 *tpc_num, u32 *broadcast_flags); | ||
279 | void (*egpc_etpc_priv_addr_table)(struct gk20a *g, u32 addr, | ||
280 | u32 gpc, u32 tpc, u32 broadcast_flags, | ||
281 | u32 *priv_addr_table, | ||
282 | u32 *priv_addr_table_index); | ||
283 | bool (*is_tpc_addr)(struct gk20a *g, u32 addr); | ||
284 | bool (*is_egpc_addr)(struct gk20a *g, u32 addr); | ||
285 | bool (*is_etpc_addr)(struct gk20a *g, u32 addr); | ||
286 | void (*get_egpc_etpc_num)(struct gk20a *g, u32 addr, | ||
287 | u32 *gpc_num, u32 *tpc_num); | ||
288 | u32 (*get_tpc_num)(struct gk20a *g, u32 addr); | ||
289 | u32 (*get_egpc_base)(struct gk20a *g); | ||
290 | void (*detect_sm_arch)(struct gk20a *g); | ||
291 | int (*add_zbc_color)(struct gk20a *g, struct gr_gk20a *gr, | ||
292 | struct zbc_entry *color_val, u32 index); | ||
293 | int (*add_zbc_depth)(struct gk20a *g, struct gr_gk20a *gr, | ||
294 | struct zbc_entry *depth_val, u32 index); | ||
295 | int (*add_zbc_s)(struct gk20a *g, struct gr_gk20a *gr, | ||
296 | struct zbc_entry *s_val, u32 index); | ||
297 | int (*zbc_set_table)(struct gk20a *g, struct gr_gk20a *gr, | ||
298 | struct zbc_entry *zbc_val); | ||
299 | int (*zbc_query_table)(struct gk20a *g, struct gr_gk20a *gr, | ||
300 | struct zbc_query_params *query_params); | ||
301 | int (*zbc_s_query_table)(struct gk20a *g, struct gr_gk20a *gr, | ||
302 | struct zbc_query_params *query_params); | ||
303 | int (*load_zbc_s_default_tbl)(struct gk20a *g, | ||
304 | struct gr_gk20a *gr); | ||
305 | int (*load_zbc_s_tbl)(struct gk20a *g, | ||
306 | struct gr_gk20a *gr); | ||
307 | void (*pmu_save_zbc)(struct gk20a *g, u32 entries); | ||
308 | int (*add_zbc)(struct gk20a *g, struct gr_gk20a *gr, | ||
309 | struct zbc_entry *zbc_val); | ||
310 | bool (*add_zbc_type_s)(struct gk20a *g, struct gr_gk20a *gr, | ||
311 | struct zbc_entry *zbc_val, int *ret_val); | ||
312 | u32 (*pagepool_default_size)(struct gk20a *g); | ||
313 | int (*init_ctx_state)(struct gk20a *g); | ||
314 | int (*alloc_gr_ctx)(struct gk20a *g, | ||
315 | struct nvgpu_gr_ctx *gr_ctx, struct vm_gk20a *vm, | ||
316 | u32 class, u32 padding); | ||
317 | void (*free_gr_ctx)(struct gk20a *g, | ||
318 | struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx); | ||
319 | void (*powergate_tpc)(struct gk20a *g); | ||
320 | void (*update_ctxsw_preemption_mode)(struct gk20a *g, | ||
321 | struct channel_gk20a *c, | ||
322 | struct nvgpu_mem *mem); | ||
323 | int (*update_smpc_ctxsw_mode)(struct gk20a *g, | ||
324 | struct channel_gk20a *c, | ||
325 | bool enable); | ||
326 | u32 (*get_hw_accessor_stream_out_mode)(void); | ||
327 | int (*update_hwpm_ctxsw_mode)(struct gk20a *g, | ||
328 | struct channel_gk20a *c, | ||
329 | u64 gpu_va, | ||
330 | u32 mode); | ||
331 | void (*init_hwpm_pmm_register)(struct gk20a *g); | ||
332 | int (*dump_gr_regs)(struct gk20a *g, | ||
333 | struct gk20a_debug_output *o); | ||
334 | int (*update_pc_sampling)(struct channel_gk20a *ch, | ||
335 | bool enable); | ||
336 | u32 (*get_max_fbps_count)(struct gk20a *g); | ||
337 | u32 (*get_fbp_en_mask)(struct gk20a *g); | ||
338 | u32 (*get_max_ltc_per_fbp)(struct gk20a *g); | ||
339 | u32 (*get_max_lts_per_ltc)(struct gk20a *g); | ||
340 | u32* (*get_rop_l2_en_mask)(struct gk20a *g); | ||
341 | void (*init_sm_dsm_reg_info)(void); | ||
342 | void (*init_ovr_sm_dsm_perf)(void); | ||
343 | int (*wait_empty)(struct gk20a *g, unsigned long duration_ms, | ||
344 | u32 expect_delay); | ||
345 | void (*init_cyclestats)(struct gk20a *g); | ||
346 | void (*enable_cde_in_fecs)(struct gk20a *g, | ||
347 | struct nvgpu_mem *mem); | ||
348 | int (*set_sm_debug_mode)(struct gk20a *g, struct channel_gk20a *ch, | ||
349 | u64 sms, bool enable); | ||
350 | void (*bpt_reg_info)(struct gk20a *g, | ||
351 | struct nvgpu_warpstate *w_state); | ||
352 | void (*get_access_map)(struct gk20a *g, | ||
353 | u32 **whitelist, int *num_entries); | ||
354 | int (*handle_fecs_error)(struct gk20a *g, | ||
355 | struct channel_gk20a *ch, | ||
356 | struct gr_gk20a_isr_data *isr_data); | ||
357 | int (*pre_process_sm_exception)(struct gk20a *g, | ||
358 | u32 gpc, u32 tpc, u32 sm, u32 global_esr, u32 warp_esr, | ||
359 | bool sm_debugger_attached, | ||
360 | struct channel_gk20a *fault_ch, | ||
361 | bool *early_exit, bool *ignore_debugger); | ||
362 | u32 (*get_sm_hww_warp_esr)(struct gk20a *g, | ||
363 | u32 gpc, u32 tpc, u32 sm); | ||
364 | u32 (*get_sm_hww_global_esr)(struct gk20a *g, | ||
365 | u32 gpc, u32 tpc, u32 sm); | ||
366 | u32 (*get_sm_no_lock_down_hww_global_esr_mask)(struct gk20a *g); | ||
367 | int (*lock_down_sm)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, | ||
368 | u32 global_esr_mask, bool check_errors); | ||
369 | int (*wait_for_sm_lock_down)(struct gk20a *g, u32 gpc, u32 tpc, | ||
370 | u32 sm, u32 global_esr_mask, bool check_errors); | ||
371 | void (*clear_sm_hww)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, | ||
372 | u32 global_esr); | ||
373 | void (*get_esr_sm_sel)(struct gk20a *g, u32 gpc, u32 tpc, | ||
374 | u32 *esr_sm_sel); | ||
375 | int (*handle_tpc_sm_ecc_exception)(struct gk20a *g, | ||
376 | u32 gpc, u32 tpc, | ||
377 | bool *post_event, struct channel_gk20a *fault_ch, | ||
378 | u32 *hww_global_esr); | ||
379 | int (*handle_sm_exception)(struct gk20a *g, | ||
380 | u32 gpc, u32 tpc, u32 sm, | ||
381 | bool *post_event, struct channel_gk20a *fault_ch, | ||
382 | u32 *hww_global_esr); | ||
383 | int (*handle_gcc_exception)(struct gk20a *g, u32 gpc, u32 tpc, | ||
384 | bool *post_event, struct channel_gk20a *fault_ch, | ||
385 | u32 *hww_global_esr); | ||
386 | int (*handle_tex_exception)(struct gk20a *g, u32 gpc, u32 tpc, | ||
387 | bool *post_event); | ||
388 | int (*handle_tpc_mpc_exception)(struct gk20a *g, | ||
389 | u32 gpc, u32 tpc, bool *post_event); | ||
390 | int (*handle_gpc_gpccs_exception)(struct gk20a *g, u32 gpc, | ||
391 | u32 gpc_exception); | ||
392 | int (*handle_gpc_gpcmmu_exception)(struct gk20a *g, u32 gpc, | ||
393 | u32 gpc_exception); | ||
394 | void (*enable_gpc_exceptions)(struct gk20a *g); | ||
395 | void (*enable_exceptions)(struct gk20a *g); | ||
396 | int (*init_ecc)(struct gk20a *g); | ||
397 | u32 (*get_lrf_tex_ltc_dram_override)(struct gk20a *g); | ||
398 | int (*record_sm_error_state)(struct gk20a *g, u32 gpc, u32 tpc, | ||
399 | u32 sm, struct channel_gk20a *fault_ch); | ||
400 | int (*update_sm_error_state)(struct gk20a *g, | ||
401 | struct channel_gk20a *ch, u32 sm_id, | ||
402 | struct nvgpu_tsg_sm_error_state *sm_error_state); | ||
403 | int (*clear_sm_error_state)(struct gk20a *g, | ||
404 | struct channel_gk20a *ch, u32 sm_id); | ||
405 | int (*suspend_contexts)(struct gk20a *g, | ||
406 | struct dbg_session_gk20a *dbg_s, | ||
407 | int *ctx_resident_ch_fd); | ||
408 | int (*resume_contexts)(struct gk20a *g, | ||
409 | struct dbg_session_gk20a *dbg_s, | ||
410 | int *ctx_resident_ch_fd); | ||
411 | int (*set_preemption_mode)(struct channel_gk20a *ch, | ||
412 | u32 graphics_preempt_mode, | ||
413 | u32 compute_preempt_mode); | ||
414 | int (*get_preemption_mode_flags)(struct gk20a *g, | ||
415 | struct nvgpu_preemption_modes_rec *preemption_modes_rec); | ||
416 | int (*set_ctxsw_preemption_mode)(struct gk20a *g, | ||
417 | struct nvgpu_gr_ctx *gr_ctx, | ||
418 | struct vm_gk20a *vm, u32 class, | ||
419 | u32 graphics_preempt_mode, | ||
420 | u32 compute_preempt_mode); | ||
421 | int (*set_boosted_ctx)(struct channel_gk20a *ch, bool boost); | ||
422 | void (*update_boosted_ctx)(struct gk20a *g, | ||
423 | struct nvgpu_mem *mem, | ||
424 | struct nvgpu_gr_ctx *gr_ctx); | ||
425 | int (*init_sm_id_table)(struct gk20a *g); | ||
426 | int (*load_smid_config)(struct gk20a *g); | ||
427 | void (*program_sm_id_numbering)(struct gk20a *g, | ||
428 | u32 gpc, u32 tpc, u32 smid); | ||
429 | void (*program_active_tpc_counts)(struct gk20a *g, u32 gpc); | ||
430 | int (*setup_rop_mapping)(struct gk20a *g, struct gr_gk20a *gr); | ||
431 | int (*init_sw_veid_bundle)(struct gk20a *g); | ||
432 | void (*program_zcull_mapping)(struct gk20a *g, | ||
433 | u32 zcull_alloc_num, u32 *zcull_map_tiles); | ||
434 | int (*commit_global_timeslice)(struct gk20a *g, | ||
435 | struct channel_gk20a *c); | ||
436 | int (*commit_inst)(struct channel_gk20a *c, u64 gpu_va); | ||
437 | void (*write_zcull_ptr)(struct gk20a *g, | ||
438 | struct nvgpu_mem *mem, u64 gpu_va); | ||
439 | void (*write_pm_ptr)(struct gk20a *g, | ||
440 | struct nvgpu_mem *mem, u64 gpu_va); | ||
441 | void (*set_preemption_buffer_va)(struct gk20a *g, | ||
442 | struct nvgpu_mem *mem, u64 gpu_va); | ||
443 | void (*load_tpc_mask)(struct gk20a *g); | ||
444 | int (*trigger_suspend)(struct gk20a *g); | ||
445 | int (*wait_for_pause)(struct gk20a *g, struct nvgpu_warpstate *w_state); | ||
446 | int (*resume_from_pause)(struct gk20a *g); | ||
447 | int (*clear_sm_errors)(struct gk20a *g); | ||
448 | u32 (*tpc_enabled_exceptions)(struct gk20a *g); | ||
449 | int (*set_czf_bypass)(struct gk20a *g, | ||
450 | struct channel_gk20a *ch); | ||
451 | void (*init_czf_bypass)(struct gk20a *g); | ||
452 | bool (*sm_debugger_attached)(struct gk20a *g); | ||
453 | void (*suspend_single_sm)(struct gk20a *g, | ||
454 | u32 gpc, u32 tpc, u32 sm, | ||
455 | u32 global_esr_mask, bool check_errors); | ||
456 | void (*suspend_all_sms)(struct gk20a *g, | ||
457 | u32 global_esr_mask, bool check_errors); | ||
458 | void (*resume_single_sm)(struct gk20a *g, | ||
459 | u32 gpc, u32 tpc, u32 sm); | ||
460 | void (*resume_all_sms)(struct gk20a *g); | ||
461 | void (*disable_rd_coalesce)(struct gk20a *g); | ||
462 | void (*init_ctxsw_hdr_data)(struct gk20a *g, | ||
463 | struct nvgpu_mem *mem); | ||
464 | void (*init_gfxp_wfi_timeout_count)(struct gk20a *g); | ||
465 | unsigned long (*get_max_gfxp_wfi_timeout_count) | ||
466 | (struct gk20a *g); | ||
467 | void (*ecc_init_scrub_reg)(struct gk20a *g); | ||
468 | u32 (*get_gpcs_swdx_dss_zbc_c_format_reg)(struct gk20a *g); | ||
469 | u32 (*get_gpcs_swdx_dss_zbc_z_format_reg)(struct gk20a *g); | ||
470 | void (*dump_ctxsw_stats)(struct gk20a *g, struct vm_gk20a *vm, | ||
471 | struct nvgpu_gr_ctx *gr_ctx); | ||
472 | void (*fecs_host_int_enable)(struct gk20a *g); | ||
473 | int (*handle_ssync_hww)(struct gk20a *g); | ||
474 | int (*handle_notify_pending)(struct gk20a *g, | ||
475 | struct gr_gk20a_isr_data *isr_data); | ||
476 | int (*handle_semaphore_pending)(struct gk20a *g, | ||
477 | struct gr_gk20a_isr_data *isr_data); | ||
478 | int (*add_ctxsw_reg_pm_fbpa)(struct gk20a *g, | ||
479 | struct ctxsw_buf_offset_map_entry *map, | ||
480 | struct aiv_list_gk20a *regs, | ||
481 | u32 *count, u32 *offset, | ||
482 | u32 max_cnt, u32 base, | ||
483 | u32 num_fbpas, u32 stride, u32 mask); | ||
484 | int (*add_ctxsw_reg_perf_pma)(struct ctxsw_buf_offset_map_entry *map, | ||
485 | struct aiv_list_gk20a *regs, | ||
486 | u32 *count, u32 *offset, | ||
487 | u32 max_cnt, u32 base, u32 mask); | ||
488 | int (*decode_priv_addr)(struct gk20a *g, u32 addr, | ||
489 | enum ctxsw_addr_type *addr_type, | ||
490 | u32 *gpc_num, u32 *tpc_num, | ||
491 | u32 *ppc_num, u32 *be_num, | ||
492 | u32 *broadcast_flags); | ||
493 | int (*create_priv_addr_table)(struct gk20a *g, | ||
494 | u32 addr, | ||
495 | u32 *priv_addr_table, | ||
496 | u32 *num_registers); | ||
497 | u32 (*get_pmm_per_chiplet_offset)(void); | ||
498 | void (*split_fbpa_broadcast_addr)(struct gk20a *g, u32 addr, | ||
499 | u32 num_fbpas, | ||
500 | u32 *priv_addr_table, | ||
501 | u32 *priv_addr_table_index); | ||
502 | u32 (*fecs_ctxsw_mailbox_size)(void); | ||
503 | int (*init_sw_bundle64)(struct gk20a *g); | ||
504 | int (*alloc_global_ctx_buffers)(struct gk20a *g); | ||
505 | int (*map_global_ctx_buffers)(struct gk20a *g, | ||
506 | struct channel_gk20a *c); | ||
507 | int (*commit_global_ctx_buffers)(struct gk20a *g, | ||
508 | struct channel_gk20a *c, bool patch); | ||
509 | u32 (*get_nonpes_aware_tpc)(struct gk20a *g, u32 gpc, u32 tpc); | ||
510 | int (*get_offset_in_gpccs_segment)(struct gk20a *g, | ||
511 | enum ctxsw_addr_type addr_type, u32 num_tpcs, | ||
512 | u32 num_ppcs, u32 reg_list_ppc_count, | ||
513 | u32 *__offset_in_segment); | ||
514 | void (*set_debug_mode)(struct gk20a *g, bool enable); | ||
515 | } gr; | ||
516 | struct { | ||
517 | void (*init_hw)(struct gk20a *g); | ||
518 | void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr); | ||
519 | void (*init_fs_state)(struct gk20a *g); | ||
520 | void (*reset)(struct gk20a *g); | ||
521 | void (*init_uncompressed_kind_map)(struct gk20a *g); | ||
522 | void (*init_kind_attr)(struct gk20a *g); | ||
523 | void (*set_mmu_page_size)(struct gk20a *g); | ||
524 | bool (*set_use_full_comp_tag_line)(struct gk20a *g); | ||
525 | u32 (*mmu_ctrl)(struct gk20a *g); | ||
526 | u32 (*mmu_debug_ctrl)(struct gk20a *g); | ||
527 | u32 (*mmu_debug_wr)(struct gk20a *g); | ||
528 | u32 (*mmu_debug_rd)(struct gk20a *g); | ||
529 | |||
530 | /* | ||
531 | * Compression tag line coverage. When mapping a compressible | ||
532 | * buffer, ctagline is increased when the virtual address | ||
533 | * crosses over the compression page boundary. | ||
534 | */ | ||
535 | unsigned int (*compression_page_size)(struct gk20a *g); | ||
536 | |||
537 | /* | ||
538 | * Minimum page size that can be used for compressible kinds. | ||
539 | */ | ||
540 | unsigned int (*compressible_page_size)(struct gk20a *g); | ||
541 | |||
542 | /* | ||
543 | * Compressible kind mappings: Mask for the virtual and physical | ||
544 | * address bits that must match. | ||
545 | */ | ||
546 | u32 (*compression_align_mask)(struct gk20a *g); | ||
547 | |||
548 | void (*dump_vpr_wpr_info)(struct gk20a *g); | ||
549 | int (*vpr_info_fetch)(struct gk20a *g); | ||
550 | void (*read_wpr_info)(struct gk20a *g, | ||
551 | struct wpr_carveout_info *inf); | ||
552 | bool (*is_debug_mode_enabled)(struct gk20a *g); | ||
553 | void (*set_debug_mode)(struct gk20a *g, bool enable); | ||
554 | int (*tlb_invalidate)(struct gk20a *g, struct nvgpu_mem *pdb); | ||
555 | void (*hub_isr)(struct gk20a *g); | ||
556 | void (*handle_replayable_fault)(struct gk20a *g); | ||
557 | int (*mem_unlock)(struct gk20a *g); | ||
558 | int (*init_nvlink)(struct gk20a *g); | ||
559 | int (*enable_nvlink)(struct gk20a *g); | ||
560 | void (*enable_hub_intr)(struct gk20a *g); | ||
561 | void (*disable_hub_intr)(struct gk20a *g); | ||
562 | int (*init_fbpa)(struct gk20a *g); | ||
563 | void (*fbpa_isr)(struct gk20a *g); | ||
564 | void (*write_mmu_fault_buffer_lo_hi)(struct gk20a *g, u32 index, | ||
565 | u32 addr_lo, u32 addr_hi); | ||
566 | void (*write_mmu_fault_buffer_get)(struct gk20a *g, u32 index, | ||
567 | u32 reg_val); | ||
568 | void (*write_mmu_fault_buffer_size)(struct gk20a *g, u32 index, | ||
569 | u32 reg_val); | ||
570 | void (*write_mmu_fault_status)(struct gk20a *g, u32 reg_val); | ||
571 | u32 (*read_mmu_fault_buffer_get)(struct gk20a *g, u32 index); | ||
572 | u32 (*read_mmu_fault_buffer_put)(struct gk20a *g, u32 index); | ||
573 | u32 (*read_mmu_fault_buffer_size)(struct gk20a *g, u32 index); | ||
574 | void (*read_mmu_fault_addr_lo_hi)(struct gk20a *g, | ||
575 | u32 *addr_lo, u32 *addr_hi); | ||
576 | void (*read_mmu_fault_inst_lo_hi)(struct gk20a *g, | ||
577 | u32 *inst_lo, u32 *inst_hi); | ||
578 | u32 (*read_mmu_fault_info)(struct gk20a *g); | ||
579 | u32 (*read_mmu_fault_status)(struct gk20a *g); | ||
580 | int (*mmu_invalidate_replay)(struct gk20a *g, | ||
581 | u32 invalidate_replay_val); | ||
582 | bool (*mmu_fault_pending)(struct gk20a *g); | ||
583 | bool (*is_fault_buf_enabled)(struct gk20a *g, u32 index); | ||
584 | void (*fault_buf_set_state_hw)(struct gk20a *g, | ||
585 | u32 index, u32 state); | ||
586 | void (*fault_buf_configure_hw)(struct gk20a *g, u32 index); | ||
587 | size_t (*get_vidmem_size)(struct gk20a *g); | ||
588 | } fb; | ||
589 | struct { | ||
590 | void (*slcg_bus_load_gating_prod)(struct gk20a *g, bool prod); | ||
591 | void (*slcg_ce2_load_gating_prod)(struct gk20a *g, bool prod); | ||
592 | void (*slcg_chiplet_load_gating_prod)(struct gk20a *g, bool prod); | ||
593 | void (*slcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod); | ||
594 | void (*slcg_fb_load_gating_prod)(struct gk20a *g, bool prod); | ||
595 | void (*slcg_fifo_load_gating_prod)(struct gk20a *g, bool prod); | ||
596 | void (*slcg_gr_load_gating_prod)(struct gk20a *g, bool prod); | ||
597 | void (*slcg_ltc_load_gating_prod)(struct gk20a *g, bool prod); | ||
598 | void (*slcg_perf_load_gating_prod)(struct gk20a *g, bool prod); | ||
599 | void (*slcg_priring_load_gating_prod)(struct gk20a *g, bool prod); | ||
600 | void (*slcg_pmu_load_gating_prod)(struct gk20a *g, bool prod); | ||
601 | void (*slcg_therm_load_gating_prod)(struct gk20a *g, bool prod); | ||
602 | void (*slcg_xbar_load_gating_prod)(struct gk20a *g, bool prod); | ||
603 | void (*blcg_bus_load_gating_prod)(struct gk20a *g, bool prod); | ||
604 | void (*blcg_ce_load_gating_prod)(struct gk20a *g, bool prod); | ||
605 | void (*blcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod); | ||
606 | void (*blcg_fb_load_gating_prod)(struct gk20a *g, bool prod); | ||
607 | void (*blcg_fifo_load_gating_prod)(struct gk20a *g, bool prod); | ||
608 | void (*blcg_gr_load_gating_prod)(struct gk20a *g, bool prod); | ||
609 | void (*blcg_ltc_load_gating_prod)(struct gk20a *g, bool prod); | ||
610 | void (*blcg_pwr_csb_load_gating_prod)(struct gk20a *g, bool prod); | ||
611 | void (*blcg_pmu_load_gating_prod)(struct gk20a *g, bool prod); | ||
612 | void (*blcg_xbar_load_gating_prod)(struct gk20a *g, bool prod); | ||
613 | void (*pg_gr_load_gating_prod)(struct gk20a *g, bool prod); | ||
614 | } clock_gating; | ||
615 | struct { | ||
616 | void (*post_events)(struct channel_gk20a *ch); | ||
617 | } debugger; | ||
618 | struct { | ||
619 | int (*setup_sw)(struct gk20a *g); | ||
620 | int (*init_fifo_setup_hw)(struct gk20a *g); | ||
621 | void (*bind_channel)(struct channel_gk20a *ch_gk20a); | ||
622 | void (*unbind_channel)(struct channel_gk20a *ch_gk20a); | ||
623 | void (*disable_channel)(struct channel_gk20a *ch); | ||
624 | void (*enable_channel)(struct channel_gk20a *ch); | ||
625 | int (*alloc_inst)(struct gk20a *g, struct channel_gk20a *ch); | ||
626 | void (*free_inst)(struct gk20a *g, struct channel_gk20a *ch); | ||
627 | int (*setup_ramfc)(struct channel_gk20a *c, u64 gpfifo_base, | ||
628 | u32 gpfifo_entries, | ||
629 | unsigned long acquire_timeout, | ||
630 | u32 flags); | ||
631 | int (*resetup_ramfc)(struct channel_gk20a *c); | ||
632 | int (*preempt_channel)(struct gk20a *g, u32 chid); | ||
633 | int (*preempt_tsg)(struct gk20a *g, u32 tsgid); | ||
634 | int (*enable_tsg)(struct tsg_gk20a *tsg); | ||
635 | int (*disable_tsg)(struct tsg_gk20a *tsg); | ||
636 | int (*tsg_verify_channel_status)(struct channel_gk20a *ch); | ||
637 | void (*tsg_verify_status_ctx_reload)(struct channel_gk20a *ch); | ||
638 | void (*tsg_verify_status_faulted)(struct channel_gk20a *ch); | ||
639 | int (*reschedule_runlist)(struct channel_gk20a *ch, | ||
640 | bool preempt_next); | ||
641 | int (*update_runlist)(struct gk20a *g, u32 runlist_id, | ||
642 | u32 chid, bool add, | ||
643 | bool wait_for_finish); | ||
644 | void (*trigger_mmu_fault)(struct gk20a *g, | ||
645 | unsigned long engine_ids); | ||
646 | void (*get_mmu_fault_info)(struct gk20a *g, u32 mmu_fault_id, | ||
647 | struct mmu_fault_info *mmfault); | ||
648 | void (*get_mmu_fault_desc)(struct mmu_fault_info *mmfault); | ||
649 | void (*get_mmu_fault_client_desc)( | ||
650 | struct mmu_fault_info *mmfault); | ||
651 | void (*get_mmu_fault_gpc_desc)(struct mmu_fault_info *mmfault); | ||
652 | void (*apply_pb_timeout)(struct gk20a *g); | ||
653 | void (*apply_ctxsw_timeout_intr)(struct gk20a *g); | ||
654 | int (*wait_engine_idle)(struct gk20a *g); | ||
655 | u32 (*get_num_fifos)(struct gk20a *g); | ||
656 | u32 (*get_pbdma_signature)(struct gk20a *g); | ||
657 | int (*set_runlist_interleave)(struct gk20a *g, u32 id, | ||
658 | u32 runlist_id, | ||
659 | u32 new_level); | ||
660 | int (*tsg_set_timeslice)(struct tsg_gk20a *tsg, u32 timeslice); | ||
661 | u32 (*default_timeslice_us)(struct gk20a *); | ||
662 | int (*force_reset_ch)(struct channel_gk20a *ch, | ||
663 | u32 err_code, bool verbose); | ||
664 | int (*engine_enum_from_type)(struct gk20a *g, u32 engine_type, | ||
665 | u32 *inst_id); | ||
666 | void (*device_info_data_parse)(struct gk20a *g, | ||
667 | u32 table_entry, u32 *inst_id, | ||
668 | u32 *pri_base, u32 *fault_id); | ||
669 | u32 (*device_info_fault_id)(u32 table_entry); | ||
670 | int (*tsg_bind_channel)(struct tsg_gk20a *tsg, | ||
671 | struct channel_gk20a *ch); | ||
672 | int (*tsg_unbind_channel)(struct channel_gk20a *ch); | ||
673 | int (*tsg_open)(struct tsg_gk20a *tsg); | ||
674 | void (*tsg_release)(struct tsg_gk20a *tsg); | ||
675 | u32 (*eng_runlist_base_size)(void); | ||
676 | int (*init_engine_info)(struct fifo_gk20a *f); | ||
677 | u32 (*runlist_entry_size)(void); | ||
678 | void (*get_tsg_runlist_entry)(struct tsg_gk20a *tsg, | ||
679 | u32 *runlist); | ||
680 | void (*get_ch_runlist_entry)(struct channel_gk20a *ch, | ||
681 | u32 *runlist); | ||
682 | u32 (*userd_gp_get)(struct gk20a *g, struct channel_gk20a *ch); | ||
683 | void (*userd_gp_put)(struct gk20a *g, struct channel_gk20a *ch); | ||
684 | u64 (*userd_pb_get)(struct gk20a *g, struct channel_gk20a *ch); | ||
685 | void (*free_channel_ctx_header)(struct channel_gk20a *ch); | ||
686 | bool (*is_fault_engine_subid_gpc)(struct gk20a *g, | ||
687 | u32 engine_subid); | ||
688 | void (*dump_pbdma_status)(struct gk20a *g, | ||
689 | struct gk20a_debug_output *o); | ||
690 | void (*dump_eng_status)(struct gk20a *g, | ||
691 | struct gk20a_debug_output *o); | ||
692 | void (*dump_channel_status_ramfc)(struct gk20a *g, | ||
693 | struct gk20a_debug_output *o, u32 chid, | ||
694 | struct ch_state *ch_state); | ||
695 | u32 (*intr_0_error_mask)(struct gk20a *g); | ||
696 | int (*is_preempt_pending)(struct gk20a *g, u32 id, | ||
697 | unsigned int id_type); | ||
698 | void (*init_pbdma_intr_descs)(struct fifo_gk20a *f); | ||
699 | int (*reset_enable_hw)(struct gk20a *g); | ||
700 | int (*setup_userd)(struct channel_gk20a *c); | ||
701 | u32 (*pbdma_acquire_val)(u64 timeout); | ||
702 | void (*teardown_ch_tsg)(struct gk20a *g, u32 act_eng_bitmask, | ||
703 | u32 id, unsigned int id_type, unsigned int rc_type, | ||
704 | struct mmu_fault_info *mmfault); | ||
705 | bool (*handle_sched_error)(struct gk20a *g); | ||
706 | bool (*handle_ctxsw_timeout)(struct gk20a *g, u32 fifo_intr); | ||
707 | unsigned int (*handle_pbdma_intr_0)(struct gk20a *g, | ||
708 | u32 pbdma_id, u32 pbdma_intr_0, | ||
709 | u32 *handled, u32 *error_notifier); | ||
710 | unsigned int (*handle_pbdma_intr_1)(struct gk20a *g, | ||
711 | u32 pbdma_id, u32 pbdma_intr_1, | ||
712 | u32 *handled, u32 *error_notifier); | ||
713 | void (*init_eng_method_buffers)(struct gk20a *g, | ||
714 | struct tsg_gk20a *tsg); | ||
715 | void (*deinit_eng_method_buffers)(struct gk20a *g, | ||
716 | struct tsg_gk20a *tsg); | ||
717 | u32 (*get_preempt_timeout)(struct gk20a *g); | ||
718 | void (*post_event_id)(struct tsg_gk20a *tsg, int event_id); | ||
719 | void (*ch_abort_clean_up)(struct channel_gk20a *ch); | ||
720 | bool (*check_tsg_ctxsw_timeout)(struct tsg_gk20a *tsg, | ||
721 | bool *verbose, u32 *ms); | ||
722 | bool (*check_ch_ctxsw_timeout)(struct channel_gk20a *ch, | ||
723 | bool *verbose, u32 *ms); | ||
724 | int (*channel_suspend)(struct gk20a *g); | ||
725 | int (*channel_resume)(struct gk20a *g); | ||
726 | void (*set_error_notifier)(struct channel_gk20a *ch, u32 error); | ||
727 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
728 | int (*alloc_syncpt_buf)(struct channel_gk20a *c, | ||
729 | u32 syncpt_id, struct nvgpu_mem *syncpt_buf); | ||
730 | void (*free_syncpt_buf)(struct channel_gk20a *c, | ||
731 | struct nvgpu_mem *syncpt_buf); | ||
732 | void (*add_syncpt_wait_cmd)(struct gk20a *g, | ||
733 | struct priv_cmd_entry *cmd, u32 off, | ||
734 | u32 id, u32 thresh, u64 gpu_va); | ||
735 | u32 (*get_syncpt_wait_cmd_size)(void); | ||
736 | void (*add_syncpt_incr_cmd)(struct gk20a *g, | ||
737 | bool wfi_cmd, struct priv_cmd_entry *cmd, | ||
738 | u32 id, u64 gpu_va); | ||
739 | u32 (*get_syncpt_incr_cmd_size)(bool wfi_cmd); | ||
740 | int (*get_sync_ro_map)(struct vm_gk20a *vm, | ||
741 | u64 *base_gpuva, u32 *sync_size); | ||
742 | u32 (*get_syncpt_incr_per_release)(void); | ||
743 | #endif | ||
744 | void (*runlist_hw_submit)(struct gk20a *g, u32 runlist_id, | ||
745 | u32 count, u32 buffer_index); | ||
746 | int (*runlist_wait_pending)(struct gk20a *g, u32 runlist_id); | ||
747 | void (*ring_channel_doorbell)(struct channel_gk20a *c); | ||
748 | u32 (*get_sema_wait_cmd_size)(void); | ||
749 | u32 (*get_sema_incr_cmd_size)(void); | ||
750 | void (*add_sema_cmd)(struct gk20a *g, | ||
751 | struct nvgpu_semaphore *s, u64 sema_va, | ||
752 | struct priv_cmd_entry *cmd, | ||
753 | u32 off, bool acquire, bool wfi); | ||
754 | } fifo; | ||
755 | struct pmu_v { | ||
756 | u32 (*get_pmu_cmdline_args_size)(struct nvgpu_pmu *pmu); | ||
757 | void (*set_pmu_cmdline_args_cpu_freq)(struct nvgpu_pmu *pmu, | ||
758 | u32 freq); | ||
759 | void (*set_pmu_cmdline_args_trace_size)(struct nvgpu_pmu *pmu, | ||
760 | u32 size); | ||
761 | void (*set_pmu_cmdline_args_trace_dma_base)( | ||
762 | struct nvgpu_pmu *pmu); | ||
763 | void (*config_pmu_cmdline_args_super_surface)( | ||
764 | struct nvgpu_pmu *pmu); | ||
765 | void (*set_pmu_cmdline_args_trace_dma_idx)( | ||
766 | struct nvgpu_pmu *pmu, u32 idx); | ||
767 | void * (*get_pmu_cmdline_args_ptr)(struct nvgpu_pmu *pmu); | ||
768 | u32 (*get_pmu_allocation_struct_size)(struct nvgpu_pmu *pmu); | ||
769 | void (*set_pmu_allocation_ptr)(struct nvgpu_pmu *pmu, | ||
770 | void **pmu_alloc_ptr, void *assign_ptr); | ||
771 | void (*pmu_allocation_set_dmem_size)(struct nvgpu_pmu *pmu, | ||
772 | void *pmu_alloc_ptr, u16 size); | ||
773 | u16 (*pmu_allocation_get_dmem_size)(struct nvgpu_pmu *pmu, | ||
774 | void *pmu_alloc_ptr); | ||
775 | u32 (*pmu_allocation_get_dmem_offset)(struct nvgpu_pmu *pmu, | ||
776 | void *pmu_alloc_ptr); | ||
777 | u32 * (*pmu_allocation_get_dmem_offset_addr)( | ||
778 | struct nvgpu_pmu *pmu, void *pmu_alloc_ptr); | ||
779 | void (*pmu_allocation_set_dmem_offset)(struct nvgpu_pmu *pmu, | ||
780 | void *pmu_alloc_ptr, u32 offset); | ||
781 | void * (*pmu_allocation_get_fb_addr)( | ||
782 | struct nvgpu_pmu *pmu, void *pmu_alloc_ptr); | ||
783 | u32 (*pmu_allocation_get_fb_size)( | ||
784 | struct nvgpu_pmu *pmu, void *pmu_alloc_ptr); | ||
785 | void (*get_pmu_init_msg_pmu_queue_params)( | ||
786 | struct nvgpu_falcon_queue *queue, u32 id, | ||
787 | void *pmu_init_msg); | ||
788 | void *(*get_pmu_msg_pmu_init_msg_ptr)( | ||
789 | struct pmu_init_msg *init); | ||
790 | u16 (*get_pmu_init_msg_pmu_sw_mg_off)( | ||
791 | union pmu_init_msg_pmu *init_msg); | ||
792 | u16 (*get_pmu_init_msg_pmu_sw_mg_size)( | ||
793 | union pmu_init_msg_pmu *init_msg); | ||
794 | u32 (*get_pmu_perfmon_cmd_start_size)(void); | ||
795 | int (*get_perfmon_cmd_start_offsetofvar)( | ||
796 | enum pmu_perfmon_cmd_start_fields field); | ||
797 | void (*perfmon_start_set_cmd_type)(struct pmu_perfmon_cmd *pc, | ||
798 | u8 value); | ||
799 | void (*perfmon_start_set_group_id)(struct pmu_perfmon_cmd *pc, | ||
800 | u8 value); | ||
801 | void (*perfmon_start_set_state_id)(struct pmu_perfmon_cmd *pc, | ||
802 | u8 value); | ||
803 | void (*perfmon_start_set_flags)(struct pmu_perfmon_cmd *pc, | ||
804 | u8 value); | ||
805 | u8 (*perfmon_start_get_flags)(struct pmu_perfmon_cmd *pc); | ||
806 | u32 (*get_pmu_perfmon_cmd_init_size)(void); | ||
807 | int (*get_perfmon_cmd_init_offsetofvar)( | ||
808 | enum pmu_perfmon_cmd_start_fields field); | ||
809 | void (*perfmon_cmd_init_set_sample_buffer)( | ||
810 | struct pmu_perfmon_cmd *pc, u16 value); | ||
811 | void (*perfmon_cmd_init_set_dec_cnt)( | ||
812 | struct pmu_perfmon_cmd *pc, u8 value); | ||
813 | void (*perfmon_cmd_init_set_base_cnt_id)( | ||
814 | struct pmu_perfmon_cmd *pc, u8 value); | ||
815 | void (*perfmon_cmd_init_set_samp_period_us)( | ||
816 | struct pmu_perfmon_cmd *pc, u32 value); | ||
817 | void (*perfmon_cmd_init_set_num_cnt)(struct pmu_perfmon_cmd *pc, | ||
818 | u8 value); | ||
819 | void (*perfmon_cmd_init_set_mov_avg)(struct pmu_perfmon_cmd *pc, | ||
820 | u8 value); | ||
821 | void *(*get_pmu_seq_in_a_ptr)( | ||
822 | struct pmu_sequence *seq); | ||
823 | void *(*get_pmu_seq_out_a_ptr)( | ||
824 | struct pmu_sequence *seq); | ||
825 | void (*set_pmu_cmdline_args_secure_mode)(struct nvgpu_pmu *pmu, | ||
826 | u32 val); | ||
827 | u32 (*get_perfmon_cntr_sz)(struct nvgpu_pmu *pmu); | ||
828 | void * (*get_perfmon_cntr_ptr)(struct nvgpu_pmu *pmu); | ||
829 | void (*set_perfmon_cntr_ut)(struct nvgpu_pmu *pmu, u16 ut); | ||
830 | void (*set_perfmon_cntr_lt)(struct nvgpu_pmu *pmu, u16 lt); | ||
831 | void (*set_perfmon_cntr_valid)(struct nvgpu_pmu *pmu, u8 val); | ||
832 | void (*set_perfmon_cntr_index)(struct nvgpu_pmu *pmu, u8 val); | ||
833 | void (*set_perfmon_cntr_group_id)(struct nvgpu_pmu *pmu, | ||
834 | u8 gid); | ||
835 | |||
836 | u8 (*pg_cmd_eng_buf_load_size)(struct pmu_pg_cmd *pg); | ||
837 | void (*pg_cmd_eng_buf_load_set_cmd_type)(struct pmu_pg_cmd *pg, | ||
838 | u8 value); | ||
839 | void (*pg_cmd_eng_buf_load_set_engine_id)(struct pmu_pg_cmd *pg, | ||
840 | u8 value); | ||
841 | void (*pg_cmd_eng_buf_load_set_buf_idx)(struct pmu_pg_cmd *pg, | ||
842 | u8 value); | ||
843 | void (*pg_cmd_eng_buf_load_set_pad)(struct pmu_pg_cmd *pg, | ||
844 | u8 value); | ||
845 | void (*pg_cmd_eng_buf_load_set_buf_size)(struct pmu_pg_cmd *pg, | ||
846 | u16 value); | ||
847 | void (*pg_cmd_eng_buf_load_set_dma_base)(struct pmu_pg_cmd *pg, | ||
848 | u32 value); | ||
849 | void (*pg_cmd_eng_buf_load_set_dma_offset)(struct pmu_pg_cmd *pg, | ||
850 | u8 value); | ||
851 | void (*pg_cmd_eng_buf_load_set_dma_idx)(struct pmu_pg_cmd *pg, | ||
852 | u8 value); | ||
853 | struct { | ||
854 | int (*boardobjgrp_pmucmd_construct_impl) | ||
855 | (struct gk20a *g, | ||
856 | struct boardobjgrp *pboardobjgrp, | ||
857 | struct boardobjgrp_pmu_cmd *cmd, u8 id, u8 msgid, | ||
858 | u8 hdrsize, u8 entrysize, u16 fbsize, u32 ss_offset, | ||
859 | u8 rpc_func_id); | ||
860 | int (*boardobjgrp_pmuset_impl)(struct gk20a *g, | ||
861 | struct boardobjgrp *pboardobjgrp); | ||
862 | int (*boardobjgrp_pmugetstatus_impl)(struct gk20a *g, | ||
863 | struct boardobjgrp *pboardobjgrp, | ||
864 | struct boardobjgrpmask *mask); | ||
865 | int (*is_boardobjgrp_pmucmd_id_valid)(struct gk20a *g, | ||
866 | struct boardobjgrp *pboardobjgrp, | ||
867 | struct boardobjgrp_pmu_cmd *cmd); | ||
868 | } boardobj; | ||
869 | struct { | ||
870 | u32 (*volt_set_voltage)(struct gk20a *g, | ||
871 | u32 logic_voltage_uv, u32 sram_voltage_uv); | ||
872 | u32 (*volt_get_voltage)(struct gk20a *g, | ||
873 | u8 volt_domain, u32 *pvoltage_uv); | ||
874 | u32 (*volt_send_load_cmd_to_pmu)(struct gk20a *g); | ||
875 | } volt; | ||
876 | struct { | ||
877 | u32 (*get_vbios_clk_domain)(u32 vbios_domain); | ||
878 | u32 (*clk_avfs_get_vin_cal_data)(struct gk20a *g, | ||
879 | struct avfsvinobjs *pvinobjs, | ||
880 | struct vin_device_v20 *pvindev); | ||
881 | u32 (*clk_vf_change_inject_data_fill)(struct gk20a *g, | ||
882 | struct nv_pmu_clk_rpc *rpccall, | ||
883 | struct set_fll_clk *setfllclk); | ||
884 | u32 (*perf_pmu_vfe_load)(struct gk20a *g); | ||
885 | u32 (*clk_set_boot_clk)(struct gk20a *g); | ||
886 | }clk; | ||
887 | } pmu_ver; | ||
888 | struct { | ||
889 | int (*get_netlist_name)(struct gk20a *g, int index, char *name); | ||
890 | bool (*is_fw_defined)(void); | ||
891 | } gr_ctx; | ||
892 | #ifdef CONFIG_GK20A_CTXSW_TRACE | ||
893 | /* | ||
894 | * Currently only supported on Linux due to the extremely tight | ||
895 | * integration with Linux device driver structure (in particular | ||
896 | * mmap). | ||
897 | */ | ||
898 | struct { | ||
899 | int (*init)(struct gk20a *g); | ||
900 | int (*max_entries)(struct gk20a *, | ||
901 | struct nvgpu_gpu_ctxsw_trace_filter *filter); | ||
902 | int (*flush)(struct gk20a *g); | ||
903 | int (*poll)(struct gk20a *g); | ||
904 | int (*enable)(struct gk20a *g); | ||
905 | int (*disable)(struct gk20a *g); | ||
906 | bool (*is_enabled)(struct gk20a *g); | ||
907 | int (*reset)(struct gk20a *g); | ||
908 | int (*bind_channel)(struct gk20a *g, struct channel_gk20a *ch); | ||
909 | int (*unbind_channel)(struct gk20a *g, | ||
910 | struct channel_gk20a *ch); | ||
911 | int (*deinit)(struct gk20a *g); | ||
912 | int (*alloc_user_buffer)(struct gk20a *g, | ||
913 | void **buf, size_t *size); | ||
914 | int (*free_user_buffer)(struct gk20a *g); | ||
915 | int (*mmap_user_buffer)(struct gk20a *g, | ||
916 | struct vm_area_struct *vma); | ||
917 | int (*set_filter)(struct gk20a *g, | ||
918 | struct nvgpu_gpu_ctxsw_trace_filter *filter); | ||
919 | } fecs_trace; | ||
920 | #endif | ||
921 | struct { | ||
922 | bool (*support_sparse)(struct gk20a *g); | ||
923 | u64 (*gmmu_map)(struct vm_gk20a *vm, | ||
924 | u64 map_offset, | ||
925 | struct nvgpu_sgt *sgt, | ||
926 | u64 buffer_offset, | ||
927 | u64 size, | ||
928 | u32 pgsz_idx, | ||
929 | u8 kind_v, | ||
930 | u32 ctag_offset, | ||
931 | u32 flags, | ||
932 | enum gk20a_mem_rw_flag rw_flag, | ||
933 | bool clear_ctags, | ||
934 | bool sparse, | ||
935 | bool priv, | ||
936 | struct vm_gk20a_mapping_batch *batch, | ||
937 | enum nvgpu_aperture aperture); | ||
938 | void (*gmmu_unmap)(struct vm_gk20a *vm, | ||
939 | u64 vaddr, | ||
940 | u64 size, | ||
941 | u32 pgsz_idx, | ||
942 | bool va_allocated, | ||
943 | enum gk20a_mem_rw_flag rw_flag, | ||
944 | bool sparse, | ||
945 | struct vm_gk20a_mapping_batch *batch); | ||
946 | int (*vm_bind_channel)(struct vm_gk20a *vm, | ||
947 | struct channel_gk20a *ch); | ||
948 | int (*fb_flush)(struct gk20a *g); | ||
949 | void (*l2_invalidate)(struct gk20a *g); | ||
950 | void (*l2_flush)(struct gk20a *g, bool invalidate); | ||
951 | void (*cbc_clean)(struct gk20a *g); | ||
952 | void (*set_big_page_size)(struct gk20a *g, | ||
953 | struct nvgpu_mem *mem, int size); | ||
954 | u32 (*get_big_page_sizes)(void); | ||
955 | u32 (*get_default_big_page_size)(void); | ||
956 | u32 (*get_iommu_bit)(struct gk20a *g); | ||
957 | int (*init_mm_setup_hw)(struct gk20a *g); | ||
958 | bool (*is_bar1_supported)(struct gk20a *g); | ||
959 | int (*init_bar2_vm)(struct gk20a *g); | ||
960 | void (*remove_bar2_vm)(struct gk20a *g); | ||
961 | const struct gk20a_mmu_level * | ||
962 | (*get_mmu_levels)(struct gk20a *g, u32 big_page_size); | ||
963 | void (*init_pdb)(struct gk20a *g, struct nvgpu_mem *inst_block, | ||
964 | struct vm_gk20a *vm); | ||
965 | u64 (*gpu_phys_addr)(struct gk20a *g, | ||
966 | struct nvgpu_gmmu_attrs *attrs, u64 phys); | ||
967 | int (*alloc_inst_block)(struct gk20a *g, | ||
968 | struct nvgpu_mem *inst_block); | ||
969 | void (*init_inst_block)(struct nvgpu_mem *inst_block, | ||
970 | struct vm_gk20a *vm, u32 big_page_size); | ||
971 | bool (*mmu_fault_pending)(struct gk20a *g); | ||
972 | void (*fault_info_mem_destroy)(struct gk20a *g); | ||
973 | void (*mmu_fault_disable_hw)(struct gk20a *g); | ||
974 | u32 (*get_kind_invalid)(void); | ||
975 | u32 (*get_kind_pitch)(void); | ||
976 | u32 (*get_flush_retries)(struct gk20a *g, | ||
977 | enum nvgpu_flush_op op); | ||
978 | } mm; | ||
979 | /* | ||
980 | * This function is called to allocate secure memory (memory | ||
981 | * that the CPU cannot see). The function should fill the | ||
982 | * context buffer descriptor (especially fields destroy, sgt, | ||
983 | * size). | ||
984 | */ | ||
985 | int (*secure_alloc)(struct gk20a *g, | ||
986 | struct gr_ctx_buffer_desc *desc, | ||
987 | size_t size); | ||
988 | struct { | ||
989 | void (*exit)(struct gk20a *g, struct nvgpu_mem *mem, | ||
990 | struct nvgpu_sgl *sgl); | ||
991 | u32 (*data032_r)(u32 i); | ||
992 | } pramin; | ||
993 | struct { | ||
994 | int (*init_therm_setup_hw)(struct gk20a *g); | ||
995 | void (*init_elcg_mode)(struct gk20a *g, u32 mode, u32 engine); | ||
996 | void (*init_blcg_mode)(struct gk20a *g, u32 mode, u32 engine); | ||
997 | int (*elcg_init_idle_filters)(struct gk20a *g); | ||
998 | #ifdef CONFIG_DEBUG_FS | ||
999 | void (*therm_debugfs_init)(struct gk20a *g); | ||
1000 | #endif | ||
1001 | int (*get_internal_sensor_curr_temp)(struct gk20a *g, u32 *temp_f24_8); | ||
1002 | void (*get_internal_sensor_limits)(s32 *max_24_8, | ||
1003 | s32 *min_24_8); | ||
1004 | u32 (*configure_therm_alert)(struct gk20a *g, s32 curr_warn_temp); | ||
1005 | } therm; | ||
1006 | struct { | ||
1007 | bool (*is_pmu_supported)(struct gk20a *g); | ||
1008 | int (*prepare_ucode)(struct gk20a *g); | ||
1009 | int (*pmu_setup_hw_and_bootstrap)(struct gk20a *g); | ||
1010 | int (*pmu_nsbootstrap)(struct nvgpu_pmu *pmu); | ||
1011 | int (*pmu_init_perfmon)(struct nvgpu_pmu *pmu); | ||
1012 | int (*pmu_perfmon_start_sampling)(struct nvgpu_pmu *pmu); | ||
1013 | int (*pmu_perfmon_stop_sampling)(struct nvgpu_pmu *pmu); | ||
1014 | int (*pmu_perfmon_get_samples_rpc)(struct nvgpu_pmu *pmu); | ||
1015 | int (*pmu_setup_elpg)(struct gk20a *g); | ||
1016 | u32 (*pmu_get_queue_head)(u32 i); | ||
1017 | u32 (*pmu_get_queue_head_size)(void); | ||
1018 | u32 (*pmu_get_queue_tail_size)(void); | ||
1019 | u32 (*pmu_get_queue_tail)(u32 i); | ||
1020 | int (*pmu_queue_head)(struct gk20a *g, | ||
1021 | struct nvgpu_falcon_queue *queue, u32 *head, bool set); | ||
1022 | int (*pmu_queue_tail)(struct gk20a *g, | ||
1023 | struct nvgpu_falcon_queue *queue, u32 *tail, bool set); | ||
1024 | void (*pmu_msgq_tail)(struct nvgpu_pmu *pmu, | ||
1025 | u32 *tail, bool set); | ||
1026 | u32 (*pmu_mutex_size)(void); | ||
1027 | int (*pmu_mutex_acquire)(struct nvgpu_pmu *pmu, | ||
1028 | u32 id, u32 *token); | ||
1029 | int (*pmu_mutex_release)(struct nvgpu_pmu *pmu, | ||
1030 | u32 id, u32 *token); | ||
1031 | bool (*pmu_is_interrupted)(struct nvgpu_pmu *pmu); | ||
1032 | void (*pmu_isr)(struct gk20a *g); | ||
1033 | void (*pmu_init_perfmon_counter)(struct gk20a *g); | ||
1034 | void (*pmu_pg_idle_counter_config)(struct gk20a *g, u32 pg_engine_id); | ||
1035 | u32 (*pmu_read_idle_counter)(struct gk20a *g, u32 counter_id); | ||
1036 | void (*pmu_reset_idle_counter)(struct gk20a *g, u32 counter_id); | ||
1037 | void (*pmu_dump_elpg_stats)(struct nvgpu_pmu *pmu); | ||
1038 | void (*pmu_dump_falcon_stats)(struct nvgpu_pmu *pmu); | ||
1039 | void (*pmu_enable_irq)(struct nvgpu_pmu *pmu, bool enable); | ||
1040 | int (*init_wpr_region)(struct gk20a *g); | ||
1041 | int (*load_lsfalcon_ucode)(struct gk20a *g, u32 falconidmask); | ||
1042 | void (*write_dmatrfbase)(struct gk20a *g, u32 addr); | ||
1043 | void (*pmu_elpg_statistics)(struct gk20a *g, u32 pg_engine_id, | ||
1044 | struct pmu_pg_stats_data *pg_stat_data); | ||
1045 | int (*pmu_pg_init_param)(struct gk20a *g, u32 pg_engine_id); | ||
1046 | int (*pmu_pg_set_sub_feature_mask)(struct gk20a *g, | ||
1047 | u32 pg_engine_id); | ||
1048 | u32 (*pmu_pg_supported_engines_list)(struct gk20a *g); | ||
1049 | u32 (*pmu_pg_engines_feature_list)(struct gk20a *g, | ||
1050 | u32 pg_engine_id); | ||
1051 | bool (*pmu_is_lpwr_feature_supported)(struct gk20a *g, | ||
1052 | u32 feature_id); | ||
1053 | int (*pmu_lpwr_enable_pg)(struct gk20a *g, bool pstate_lock); | ||
1054 | int (*pmu_lpwr_disable_pg)(struct gk20a *g, bool pstate_lock); | ||
1055 | u32 (*pmu_pg_param_post_init)(struct gk20a *g); | ||
1056 | void (*dump_secure_fuses)(struct gk20a *g); | ||
1057 | int (*reset_engine)(struct gk20a *g, bool do_reset); | ||
1058 | bool (*is_engine_in_reset)(struct gk20a *g); | ||
1059 | int (*falcon_wait_for_halt)(struct gk20a *g, | ||
1060 | unsigned int timeout); | ||
1061 | int (*falcon_clear_halt_interrupt_status)(struct gk20a *g, | ||
1062 | unsigned int timeout); | ||
1063 | int (*init_falcon_setup_hw)(struct gk20a *g, | ||
1064 | void *desc, u32 bl_sz); | ||
1065 | bool (*is_lazy_bootstrap)(u32 falcon_id); | ||
1066 | bool (*is_priv_load)(u32 falcon_id); | ||
1067 | void (*get_wpr)(struct gk20a *g, struct wpr_carveout_info *inf); | ||
1068 | int (*alloc_blob_space)(struct gk20a *g, | ||
1069 | size_t size, struct nvgpu_mem *mem); | ||
1070 | int (*pmu_populate_loader_cfg)(struct gk20a *g, | ||
1071 | void *lsfm, u32 *p_bl_gen_desc_size); | ||
1072 | int (*flcn_populate_bl_dmem_desc)(struct gk20a *g, | ||
1073 | void *lsfm, u32 *p_bl_gen_desc_size, u32 falconid); | ||
1074 | void (*handle_ext_irq)(struct gk20a *g, u32 intr); | ||
1075 | void (*set_irqmask)(struct gk20a *g); | ||
1076 | void (*update_lspmu_cmdline_args)(struct gk20a *g); | ||
1077 | void (*setup_apertures)(struct gk20a *g); | ||
1078 | u32 (*get_irqdest)(struct gk20a *g); | ||
1079 | int (*alloc_super_surface)(struct gk20a *g, | ||
1080 | struct nvgpu_mem *super_surface, u32 size); | ||
1081 | bool (*is_debug_mode_enabled)(struct gk20a *g); | ||
1082 | } pmu; | ||
1083 | struct { | ||
1084 | int (*init_debugfs)(struct gk20a *g); | ||
1085 | void (*disable_slowboot)(struct gk20a *g); | ||
1086 | int (*init_clk_support)(struct gk20a *g); | ||
1087 | int (*suspend_clk_support)(struct gk20a *g); | ||
1088 | u32 (*get_crystal_clk_hz)(struct gk20a *g); | ||
1089 | int (*clk_domain_get_f_points)(struct gk20a *g, | ||
1090 | u32 clkapidomain, u32 *pfpointscount, | ||
1091 | u16 *pfreqpointsinmhz); | ||
1092 | unsigned long (*measure_freq)(struct gk20a *g, u32 api_domain); | ||
1093 | u32 (*get_rate_cntr)(struct gk20a *g, struct namemap_cfg *c); | ||
1094 | unsigned long (*get_rate)(struct gk20a *g, u32 api_domain); | ||
1095 | int (*set_rate)(struct gk20a *g, u32 api_domain, unsigned long rate); | ||
1096 | unsigned long (*get_fmax_at_vmin_safe)(struct gk20a *g); | ||
1097 | u32 (*get_ref_clock_rate)(struct gk20a *g); | ||
1098 | int (*predict_mv_at_hz_cur_tfloor)(struct clk_gk20a *clk, | ||
1099 | unsigned long rate); | ||
1100 | unsigned long (*get_maxrate)(struct gk20a *g, u32 api_domain); | ||
1101 | int (*prepare_enable)(struct clk_gk20a *clk); | ||
1102 | void (*disable_unprepare)(struct clk_gk20a *clk); | ||
1103 | int (*get_voltage)(struct clk_gk20a *clk, u64 *val); | ||
1104 | int (*get_gpcclk_clock_counter)(struct clk_gk20a *clk, u64 *val); | ||
1105 | int (*pll_reg_write)(struct gk20a *g, u32 reg, u32 val); | ||
1106 | int (*get_pll_debug_data)(struct gk20a *g, | ||
1107 | struct nvgpu_clk_pll_debug_data *d); | ||
1108 | int (*mclk_init)(struct gk20a *g); | ||
1109 | void (*mclk_deinit)(struct gk20a *g); | ||
1110 | int (*mclk_change)(struct gk20a *g, u16 val); | ||
1111 | bool split_rail_support; | ||
1112 | bool support_clk_freq_controller; | ||
1113 | bool support_pmgr_domain; | ||
1114 | bool support_lpwr_pg; | ||
1115 | } clk; | ||
1116 | struct { | ||
1117 | int (*arbiter_clk_init)(struct gk20a *g); | ||
1118 | u32 (*get_arbiter_clk_domains)(struct gk20a *g); | ||
1119 | int (*get_arbiter_clk_range)(struct gk20a *g, u32 api_domain, | ||
1120 | u16 *min_mhz, u16 *max_mhz); | ||
1121 | int (*get_arbiter_clk_default)(struct gk20a *g, u32 api_domain, | ||
1122 | u16 *default_mhz); | ||
1123 | void (*clk_arb_run_arbiter_cb)(struct nvgpu_clk_arb *arb); | ||
1124 | /* This function is inherently unsafe to call while | ||
1125 | * arbiter is running arbiter must be blocked | ||
1126 | * before calling this function */ | ||
1127 | int (*get_current_pstate)(struct gk20a *g); | ||
1128 | void (*clk_arb_cleanup)(struct nvgpu_clk_arb *arb); | ||
1129 | } clk_arb; | ||
1130 | struct { | ||
1131 | int (*handle_pmu_perf_event)(struct gk20a *g, void *pmu_msg); | ||
1132 | } perf; | ||
1133 | struct { | ||
1134 | int (*exec_regops)(struct dbg_session_gk20a *dbg_s, | ||
1135 | struct nvgpu_dbg_reg_op *ops, | ||
1136 | u64 num_ops, | ||
1137 | bool *is_current_ctx); | ||
1138 | const struct regop_offset_range* ( | ||
1139 | *get_global_whitelist_ranges)(void); | ||
1140 | u64 (*get_global_whitelist_ranges_count)(void); | ||
1141 | const struct regop_offset_range* ( | ||
1142 | *get_context_whitelist_ranges)(void); | ||
1143 | u64 (*get_context_whitelist_ranges_count)(void); | ||
1144 | const u32* (*get_runcontrol_whitelist)(void); | ||
1145 | u64 (*get_runcontrol_whitelist_count)(void); | ||
1146 | const u32* (*get_qctl_whitelist)(void); | ||
1147 | u64 (*get_qctl_whitelist_count)(void); | ||
1148 | } regops; | ||
1149 | struct { | ||
1150 | void (*intr_mask)(struct gk20a *g); | ||
1151 | void (*intr_enable)(struct gk20a *g); | ||
1152 | void (*intr_unit_config)(struct gk20a *g, | ||
1153 | bool enable, bool is_stalling, u32 unit); | ||
1154 | void (*isr_stall)(struct gk20a *g); | ||
1155 | bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); | ||
1156 | bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr); | ||
1157 | bool (*is_stall_and_eng_intr_pending)(struct gk20a *g, | ||
1158 | u32 act_eng_id, u32 *eng_intr_pending); | ||
1159 | u32 (*intr_stall)(struct gk20a *g); | ||
1160 | void (*intr_stall_pause)(struct gk20a *g); | ||
1161 | void (*intr_stall_resume)(struct gk20a *g); | ||
1162 | u32 (*intr_nonstall)(struct gk20a *g); | ||
1163 | void (*intr_nonstall_pause)(struct gk20a *g); | ||
1164 | void (*intr_nonstall_resume)(struct gk20a *g); | ||
1165 | u32 (*isr_nonstall)(struct gk20a *g); | ||
1166 | void (*enable)(struct gk20a *g, u32 units); | ||
1167 | void (*disable)(struct gk20a *g, u32 units); | ||
1168 | void (*reset)(struct gk20a *g, u32 units); | ||
1169 | u32 (*boot_0)(struct gk20a *g, u32 *arch, u32 *impl, u32 *rev); | ||
1170 | bool (*is_intr1_pending)(struct gk20a *g, enum nvgpu_unit unit, u32 mc_intr_1); | ||
1171 | void (*log_pending_intrs)(struct gk20a *g); | ||
1172 | } mc; | ||
1173 | struct { | ||
1174 | void (*show_dump)(struct gk20a *g, | ||
1175 | struct gk20a_debug_output *o); | ||
1176 | } debug; | ||
1177 | struct { | ||
1178 | int (*dbg_set_powergate)(struct dbg_session_gk20a *dbg_s, | ||
1179 | bool disable_powergate); | ||
1180 | bool (*check_and_set_global_reservation)( | ||
1181 | struct dbg_session_gk20a *dbg_s, | ||
1182 | struct dbg_profiler_object_data *prof_obj); | ||
1183 | bool (*check_and_set_context_reservation)( | ||
1184 | struct dbg_session_gk20a *dbg_s, | ||
1185 | struct dbg_profiler_object_data *prof_obj); | ||
1186 | void (*release_profiler_reservation)( | ||
1187 | struct dbg_session_gk20a *dbg_s, | ||
1188 | struct dbg_profiler_object_data *prof_obj); | ||
1189 | int (*perfbuffer_enable)(struct gk20a *g, u64 offset, u32 size); | ||
1190 | int (*perfbuffer_disable)(struct gk20a *g); | ||
1191 | } dbg_session_ops; | ||
1192 | |||
1193 | int (*get_litter_value)(struct gk20a *g, int value); | ||
1194 | int (*chip_init_gpu_characteristics)(struct gk20a *g); | ||
1195 | |||
1196 | struct { | ||
1197 | void (*init_hw)(struct gk20a *g); | ||
1198 | void (*isr)(struct gk20a *g); | ||
1199 | int (*bar1_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst); | ||
1200 | int (*bar2_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst); | ||
1201 | u32 (*set_bar0_window)(struct gk20a *g, struct nvgpu_mem *mem, | ||
1202 | struct nvgpu_sgt *sgt, struct nvgpu_sgl *sgl, | ||
1203 | u32 w); | ||
1204 | u32 (*read_sw_scratch)(struct gk20a *g, u32 index); | ||
1205 | void (*write_sw_scratch)(struct gk20a *g, u32 index, u32 val); | ||
1206 | } bus; | ||
1207 | |||
1208 | struct { | ||
1209 | void (*isr)(struct gk20a *g); | ||
1210 | int (*read_ptimer)(struct gk20a *g, u64 *value); | ||
1211 | int (*get_timestamps_zipper)(struct gk20a *g, | ||
1212 | u32 source_id, u32 count, | ||
1213 | struct nvgpu_cpu_time_correlation_sample *); | ||
1214 | } ptimer; | ||
1215 | |||
1216 | struct { | ||
1217 | int (*init)(struct gk20a *g); | ||
1218 | int (*preos_wait_for_halt)(struct gk20a *g); | ||
1219 | void (*preos_reload_check)(struct gk20a *g); | ||
1220 | int (*devinit)(struct gk20a *g); | ||
1221 | int (*preos)(struct gk20a *g); | ||
1222 | int (*verify_devinit)(struct gk20a *g); | ||
1223 | } bios; | ||
1224 | |||
1225 | #if defined(CONFIG_GK20A_CYCLE_STATS) | ||
1226 | struct { | ||
1227 | int (*enable_snapshot)(struct channel_gk20a *ch, | ||
1228 | struct gk20a_cs_snapshot_client *client); | ||
1229 | void (*disable_snapshot)(struct gr_gk20a *gr); | ||
1230 | int (*check_data_available)(struct channel_gk20a *ch, | ||
1231 | u32 *pending, | ||
1232 | bool *hw_overflow); | ||
1233 | void (*set_handled_snapshots)(struct gk20a *g, u32 num); | ||
1234 | u32 (*allocate_perfmon_ids)(struct gk20a_cs_snapshot *data, | ||
1235 | u32 count); | ||
1236 | u32 (*release_perfmon_ids)(struct gk20a_cs_snapshot *data, | ||
1237 | u32 start, | ||
1238 | u32 count); | ||
1239 | int (*detach_snapshot)(struct channel_gk20a *ch, | ||
1240 | struct gk20a_cs_snapshot_client *client); | ||
1241 | bool (*get_overflow_status)(struct gk20a *g); | ||
1242 | u32 (*get_pending_snapshots)(struct gk20a *g); | ||
1243 | } css; | ||
1244 | #endif | ||
1245 | struct { | ||
1246 | int (*get_speed)(struct gk20a *g, u32 *xve_link_speed); | ||
1247 | int (*set_speed)(struct gk20a *g, u32 xve_link_speed); | ||
1248 | void (*available_speeds)(struct gk20a *g, u32 *speed_mask); | ||
1249 | u32 (*xve_readl)(struct gk20a *g, u32 reg); | ||
1250 | void (*xve_writel)(struct gk20a *g, u32 reg, u32 val); | ||
1251 | void (*disable_aspm)(struct gk20a *g); | ||
1252 | void (*reset_gpu)(struct gk20a *g); | ||
1253 | #if defined(CONFIG_PCI_MSI) | ||
1254 | void (*rearm_msi)(struct gk20a *g); | ||
1255 | #endif | ||
1256 | void (*enable_shadow_rom)(struct gk20a *g); | ||
1257 | void (*disable_shadow_rom)(struct gk20a *g); | ||
1258 | u32 (*get_link_control_status)(struct gk20a *g); | ||
1259 | } xve; | ||
1260 | struct { | ||
1261 | int (*falcon_hal_sw_init)(struct nvgpu_falcon *flcn); | ||
1262 | } falcon; | ||
1263 | struct { | ||
1264 | void (*enable_priv_ring)(struct gk20a *g); | ||
1265 | void (*isr)(struct gk20a *g); | ||
1266 | void (*decode_error_code)(struct gk20a *g, u32 error_code); | ||
1267 | void (*set_ppriv_timeout_settings)(struct gk20a *g); | ||
1268 | u32 (*enum_ltc)(struct gk20a *g); | ||
1269 | } priv_ring; | ||
1270 | struct { | ||
1271 | int (*check_priv_security)(struct gk20a *g); | ||
1272 | bool (*is_opt_ecc_enable)(struct gk20a *g); | ||
1273 | bool (*is_opt_feature_override_disable)(struct gk20a *g); | ||
1274 | u32 (*fuse_status_opt_fbio)(struct gk20a *g); | ||
1275 | u32 (*fuse_status_opt_fbp)(struct gk20a *g); | ||
1276 | u32 (*fuse_status_opt_rop_l2_fbp)(struct gk20a *g, u32 fbp); | ||
1277 | u32 (*fuse_status_opt_tpc_gpc)(struct gk20a *g, u32 gpc); | ||
1278 | void (*fuse_ctrl_opt_tpc_gpc)(struct gk20a *g, u32 gpc, u32 val); | ||
1279 | u32 (*fuse_opt_sec_debug_en)(struct gk20a *g); | ||
1280 | u32 (*fuse_opt_priv_sec_en)(struct gk20a *g); | ||
1281 | u32 (*read_vin_cal_fuse_rev)(struct gk20a *g); | ||
1282 | u32 (*read_vin_cal_slope_intercept_fuse)(struct gk20a *g, | ||
1283 | u32 vin_id, u32 *slope, | ||
1284 | u32 *intercept); | ||
1285 | u32 (*read_vin_cal_gain_offset_fuse)(struct gk20a *g, | ||
1286 | u32 vin_id, s8 *gain, | ||
1287 | s8 *offset); | ||
1288 | } fuse; | ||
1289 | struct { | ||
1290 | int (*init)(struct gk20a *g); | ||
1291 | int (*discover_ioctrl)(struct gk20a *g); | ||
1292 | int (*discover_link)(struct gk20a *g); | ||
1293 | int (*isr)(struct gk20a *g); | ||
1294 | int (*rxdet)(struct gk20a *g, u32 link_id); | ||
1295 | int (*setup_pll)(struct gk20a *g, unsigned long link_mask); | ||
1296 | int (*minion_data_ready_en)(struct gk20a *g, | ||
1297 | unsigned long link_mask, bool sync); | ||
1298 | void (*get_connected_link_mask)(u32 *link_mask); | ||
1299 | void (*set_sw_war)(struct gk20a *g, u32 link_id); | ||
1300 | /* API */ | ||
1301 | int (*link_early_init)(struct gk20a *g, unsigned long mask); | ||
1302 | u32 (*link_get_mode)(struct gk20a *g, u32 link_id); | ||
1303 | u32 (*link_get_state)(struct gk20a *g, u32 link_id); | ||
1304 | int (*link_set_mode)(struct gk20a *g, u32 link_id, u32 mode); | ||
1305 | u32 (*get_sublink_mode)(struct gk20a *g, u32 link_id, | ||
1306 | bool is_rx_sublink); | ||
1307 | u32 (*get_rx_sublink_state)(struct gk20a *g, u32 link_id); | ||
1308 | u32 (*get_tx_sublink_state)(struct gk20a *g, u32 link_id); | ||
1309 | int (*set_sublink_mode)(struct gk20a *g, u32 link_id, | ||
1310 | bool is_rx_sublink, u32 mode); | ||
1311 | int (*interface_init)(struct gk20a *g); | ||
1312 | int (*interface_disable)(struct gk20a *g); | ||
1313 | int (*reg_init)(struct gk20a *g); | ||
1314 | int (*shutdown)(struct gk20a *g); | ||
1315 | int (*early_init)(struct gk20a *g); | ||
1316 | } nvlink; | ||
1317 | struct { | ||
1318 | u32 (*get_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g); | ||
1319 | void (*set_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g, u32 val); | ||
1320 | u32 (*get_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g); | ||
1321 | void (*set_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g, u32 val); | ||
1322 | } top; | ||
1323 | void (*semaphore_wakeup)(struct gk20a *g, bool post_events); | ||
1324 | }; | ||
1325 | |||
1326 | struct nvgpu_bios_ucode { | ||
1327 | u8 *bootloader; | ||
1328 | u32 bootloader_phys_base; | ||
1329 | u32 bootloader_size; | ||
1330 | u8 *ucode; | ||
1331 | u32 phys_base; | ||
1332 | u32 size; | ||
1333 | u8 *dmem; | ||
1334 | u32 dmem_phys_base; | ||
1335 | u32 dmem_size; | ||
1336 | u32 code_entry_point; | ||
1337 | }; | ||
1338 | |||
1339 | struct nvgpu_bios { | ||
1340 | u32 vbios_version; | ||
1341 | u8 vbios_oem_version; | ||
1342 | |||
1343 | u8 *data; | ||
1344 | size_t size; | ||
1345 | |||
1346 | struct nvgpu_bios_ucode devinit; | ||
1347 | struct nvgpu_bios_ucode preos; | ||
1348 | |||
1349 | u8 *devinit_tables; | ||
1350 | u32 devinit_tables_size; | ||
1351 | u8 *bootscripts; | ||
1352 | u32 bootscripts_size; | ||
1353 | |||
1354 | u8 mem_strap_data_count; | ||
1355 | u16 mem_strap_xlat_tbl_ptr; | ||
1356 | |||
1357 | u32 condition_table_ptr; | ||
1358 | |||
1359 | u32 devinit_tables_phys_base; | ||
1360 | u32 devinit_script_phys_base; | ||
1361 | |||
1362 | struct bit_token *perf_token; | ||
1363 | struct bit_token *clock_token; | ||
1364 | struct bit_token *virt_token; | ||
1365 | u32 expansion_rom_offset; | ||
1366 | |||
1367 | u32 nvlink_config_data_offset; | ||
1368 | }; | ||
1369 | |||
1370 | struct nvgpu_gpu_params { | ||
1371 | /* GPU architecture ID */ | ||
1372 | u32 gpu_arch; | ||
1373 | /* GPU implementation ID */ | ||
1374 | u32 gpu_impl; | ||
1375 | /* GPU revision ID */ | ||
1376 | u32 gpu_rev; | ||
1377 | /* sm version */ | ||
1378 | u32 sm_arch_sm_version; | ||
1379 | /* sm instruction set */ | ||
1380 | u32 sm_arch_spa_version; | ||
1381 | u32 sm_arch_warp_count; | ||
1382 | }; | ||
1383 | |||
1384 | struct gk20a { | ||
1385 | void (*free)(struct gk20a *g); | ||
1386 | struct nvgpu_nvhost_dev *nvhost_dev; | ||
1387 | |||
1388 | /* | ||
1389 | * Used by <nvgpu/enabled.h>. Do not access directly! | ||
1390 | */ | ||
1391 | unsigned long *enabled_flags; | ||
1392 | |||
1393 | nvgpu_atomic_t usage_count; | ||
1394 | |||
1395 | struct nvgpu_mutex ctxsw_disable_lock; | ||
1396 | int ctxsw_disable_count; | ||
1397 | |||
1398 | struct nvgpu_ref refcount; | ||
1399 | |||
1400 | const char *name; | ||
1401 | |||
1402 | bool gpu_reset_done; | ||
1403 | bool power_on; | ||
1404 | bool suspended; | ||
1405 | bool sw_ready; | ||
1406 | |||
1407 | u64 log_mask; | ||
1408 | u32 log_trace; | ||
1409 | |||
1410 | struct nvgpu_mutex tpc_pg_lock; | ||
1411 | |||
1412 | struct nvgpu_gpu_params params; | ||
1413 | |||
1414 | /* | ||
1415 | * Guards access to hardware when usual gk20a_{busy,idle} are skipped | ||
1416 | * for submits and held for channel lifetime but dropped for an ongoing | ||
1417 | * gk20a_do_idle(). | ||
1418 | */ | ||
1419 | struct nvgpu_rwsem deterministic_busy; | ||
1420 | |||
1421 | struct nvgpu_falcon pmu_flcn; | ||
1422 | struct nvgpu_falcon sec2_flcn; | ||
1423 | struct nvgpu_falcon fecs_flcn; | ||
1424 | struct nvgpu_falcon gpccs_flcn; | ||
1425 | struct nvgpu_falcon nvdec_flcn; | ||
1426 | struct nvgpu_falcon minion_flcn; | ||
1427 | struct clk_gk20a clk; | ||
1428 | struct fifo_gk20a fifo; | ||
1429 | struct nvgpu_nvlink_dev nvlink; | ||
1430 | struct gr_gk20a gr; | ||
1431 | struct sim_nvgpu *sim; | ||
1432 | struct mm_gk20a mm; | ||
1433 | struct nvgpu_pmu pmu; | ||
1434 | struct acr_desc acr; | ||
1435 | struct nvgpu_ecc ecc; | ||
1436 | struct clk_pmupstate clk_pmu; | ||
1437 | struct perf_pmupstate perf_pmu; | ||
1438 | struct pmgr_pmupstate pmgr_pmu; | ||
1439 | struct therm_pmupstate therm_pmu; | ||
1440 | |||
1441 | #ifdef CONFIG_DEBUG_FS | ||
1442 | struct railgate_stats pstats; | ||
1443 | #endif | ||
1444 | u32 gr_idle_timeout_default; | ||
1445 | bool timeouts_disabled_by_user; | ||
1446 | unsigned int ch_wdt_timeout_ms; | ||
1447 | u32 fifo_eng_timeout_us; | ||
1448 | |||
1449 | struct nvgpu_mutex poweron_lock; | ||
1450 | struct nvgpu_mutex poweroff_lock; | ||
1451 | |||
1452 | /* Channel priorities */ | ||
1453 | u32 timeslice_low_priority_us; | ||
1454 | u32 timeslice_medium_priority_us; | ||
1455 | u32 timeslice_high_priority_us; | ||
1456 | u32 min_timeslice_us; | ||
1457 | u32 max_timeslice_us; | ||
1458 | bool runlist_interleave; | ||
1459 | |||
1460 | bool slcg_enabled; | ||
1461 | bool blcg_enabled; | ||
1462 | bool elcg_enabled; | ||
1463 | bool elpg_enabled; | ||
1464 | bool aelpg_enabled; | ||
1465 | bool can_elpg; | ||
1466 | bool mscg_enabled; | ||
1467 | bool forced_idle; | ||
1468 | bool forced_reset; | ||
1469 | bool allow_all; | ||
1470 | |||
1471 | u32 ptimer_src_freq; | ||
1472 | |||
1473 | int railgate_delay; | ||
1474 | u8 ldiv_slowdown_factor; | ||
1475 | unsigned int aggressive_sync_destroy_thresh; | ||
1476 | bool aggressive_sync_destroy; | ||
1477 | |||
1478 | bool has_syncpoints; | ||
1479 | /* Debugfs knob for forcing syncpt support off in runtime. */ | ||
1480 | u32 disable_syncpoints; | ||
1481 | |||
1482 | bool support_pmu; | ||
1483 | u32 bootstrap_owner; | ||
1484 | |||
1485 | bool is_virtual; | ||
1486 | |||
1487 | bool has_cde; | ||
1488 | |||
1489 | u32 emc3d_ratio; | ||
1490 | |||
1491 | struct nvgpu_spinlock ltc_enabled_lock; | ||
1492 | |||
1493 | struct gk20a_ctxsw_ucode_info ctxsw_ucode_info; | ||
1494 | |||
1495 | /* | ||
1496 | * A group of semaphore pools. One for each channel. | ||
1497 | */ | ||
1498 | struct nvgpu_semaphore_sea *sema_sea; | ||
1499 | |||
1500 | /* held while manipulating # of debug/profiler sessions present */ | ||
1501 | /* also prevents debug sessions from attaching until released */ | ||
1502 | struct nvgpu_mutex dbg_sessions_lock; | ||
1503 | int dbg_powergating_disabled_refcount; /*refcount for pg disable */ | ||
1504 | /*refcount for timeout disable */ | ||
1505 | nvgpu_atomic_t timeouts_disabled_refcount; | ||
1506 | |||
1507 | /* must have dbg_sessions_lock before use */ | ||
1508 | struct nvgpu_dbg_reg_op *dbg_regops_tmp_buf; | ||
1509 | u32 dbg_regops_tmp_buf_ops; | ||
1510 | |||
1511 | /* For perfbuf mapping */ | ||
1512 | struct { | ||
1513 | struct dbg_session_gk20a *owner; | ||
1514 | u64 offset; | ||
1515 | } perfbuf; | ||
1516 | |||
1517 | /* For profiler reservations */ | ||
1518 | struct nvgpu_list_node profiler_objects; | ||
1519 | bool global_profiler_reservation_held; | ||
1520 | int profiler_reservation_count; | ||
1521 | |||
1522 | void (*remove_support)(struct gk20a *); | ||
1523 | |||
1524 | u64 pg_ingating_time_us; | ||
1525 | u64 pg_ungating_time_us; | ||
1526 | u32 pg_gating_cnt; | ||
1527 | |||
1528 | struct nvgpu_spinlock mc_enable_lock; | ||
1529 | |||
1530 | struct gk20a_as as; | ||
1531 | |||
1532 | struct nvgpu_mutex client_lock; | ||
1533 | int client_refcount; /* open channels and ctrl nodes */ | ||
1534 | |||
1535 | struct gpu_ops ops; | ||
1536 | u32 mc_intr_mask_restore[4]; | ||
1537 | /*used for change of enum zbc update cmd id from ver 0 to ver1*/ | ||
1538 | u32 pmu_ver_cmd_id_zbc_table_update; | ||
1539 | u32 pmu_lsf_pmu_wpr_init_done; | ||
1540 | u32 pmu_lsf_loaded_falcon_id; | ||
1541 | |||
1542 | int irqs_enabled; | ||
1543 | int irq_stall; /* can be same as irq_nonstall in case of PCI */ | ||
1544 | int irq_nonstall; | ||
1545 | u32 max_ltc_count; | ||
1546 | u32 ltc_count; | ||
1547 | u32 ltc_streamid; | ||
1548 | |||
1549 | struct gk20a_worker { | ||
1550 | struct nvgpu_thread poll_task; | ||
1551 | nvgpu_atomic_t put; | ||
1552 | struct nvgpu_cond wq; | ||
1553 | struct nvgpu_list_node items; | ||
1554 | struct nvgpu_spinlock items_lock; | ||
1555 | struct nvgpu_mutex start_lock; | ||
1556 | } channel_worker, clk_arb_worker; | ||
1557 | |||
1558 | struct { | ||
1559 | void (*open)(struct channel_gk20a *ch); | ||
1560 | void (*close)(struct channel_gk20a *ch); | ||
1561 | void (*work_completion_signal)(struct channel_gk20a *ch); | ||
1562 | void (*work_completion_cancel_sync)(struct channel_gk20a *ch); | ||
1563 | bool (*os_fence_framework_inst_exists)(struct channel_gk20a *ch); | ||
1564 | int (*init_os_fence_framework)( | ||
1565 | struct channel_gk20a *ch, const char *fmt, ...); | ||
1566 | void (*signal_os_fence_framework)(struct channel_gk20a *ch); | ||
1567 | void (*destroy_os_fence_framework)(struct channel_gk20a *ch); | ||
1568 | int (*copy_user_gpfifo)(struct nvgpu_gpfifo_entry *dest, | ||
1569 | struct nvgpu_gpfifo_userdata userdata, | ||
1570 | u32 start, u32 length); | ||
1571 | int (*alloc_usermode_buffers)(struct channel_gk20a *c, | ||
1572 | struct nvgpu_gpfifo_args *gpfifo_args); | ||
1573 | } os_channel; | ||
1574 | |||
1575 | struct gk20a_scale_profile *scale_profile; | ||
1576 | unsigned long last_freq; | ||
1577 | |||
1578 | struct gk20a_ctxsw_trace *ctxsw_trace; | ||
1579 | struct gk20a_fecs_trace *fecs_trace; | ||
1580 | |||
1581 | bool mmu_debug_ctrl; | ||
1582 | |||
1583 | u32 tpc_fs_mask_user; | ||
1584 | |||
1585 | u32 tpc_pg_mask; | ||
1586 | bool can_tpc_powergate; | ||
1587 | |||
1588 | u32 valid_tpc_mask; | ||
1589 | |||
1590 | struct nvgpu_bios bios; | ||
1591 | bool bios_is_init; | ||
1592 | |||
1593 | struct nvgpu_clk_arb *clk_arb; | ||
1594 | |||
1595 | struct nvgpu_mutex clk_arb_enable_lock; | ||
1596 | |||
1597 | struct gk20a_ce_app ce_app; | ||
1598 | |||
1599 | bool ltc_intr_en_illegal_compstat; | ||
1600 | |||
1601 | /* PCI device identifier */ | ||
1602 | u16 pci_vendor_id, pci_device_id; | ||
1603 | u16 pci_subsystem_vendor_id, pci_subsystem_device_id; | ||
1604 | u16 pci_class; | ||
1605 | u8 pci_revision; | ||
1606 | |||
1607 | /* | ||
1608 | * PCI power management: i2c device index, port and address for | ||
1609 | * INA3221. | ||
1610 | */ | ||
1611 | u32 ina3221_dcb_index; | ||
1612 | u32 ina3221_i2c_address; | ||
1613 | u32 ina3221_i2c_port; | ||
1614 | bool hardcode_sw_threshold; | ||
1615 | |||
1616 | /* PCIe power states. */ | ||
1617 | bool xve_l0s; | ||
1618 | bool xve_l1; | ||
1619 | |||
1620 | /* Current warning temp in sfxp24.8 */ | ||
1621 | s32 curr_warn_temp; | ||
1622 | |||
1623 | #if defined(CONFIG_PCI_MSI) | ||
1624 | /* Check if msi is enabled */ | ||
1625 | bool msi_enabled; | ||
1626 | #endif | ||
1627 | #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE | ||
1628 | struct nvgpu_mem_alloc_tracker *vmallocs; | ||
1629 | struct nvgpu_mem_alloc_tracker *kmallocs; | ||
1630 | #endif | ||
1631 | |||
1632 | /* The minimum VBIOS version supported */ | ||
1633 | u32 vbios_min_version; | ||
1634 | |||
1635 | /* memory training sequence and mclk switch scripts */ | ||
1636 | u32 mem_config_idx; | ||
1637 | |||
1638 | u64 dma_memory_used; | ||
1639 | |||
1640 | #if defined(CONFIG_TEGRA_GK20A_NVHOST) | ||
1641 | u64 syncpt_unit_base; | ||
1642 | size_t syncpt_unit_size; | ||
1643 | u32 syncpt_size; | ||
1644 | #endif | ||
1645 | struct nvgpu_mem syncpt_mem; | ||
1646 | |||
1647 | struct nvgpu_list_node boardobj_head; | ||
1648 | struct nvgpu_list_node boardobjgrp_head; | ||
1649 | }; | ||
1650 | |||
1651 | static inline bool nvgpu_is_timeouts_enabled(struct gk20a *g) | ||
1652 | { | ||
1653 | return nvgpu_atomic_read(&g->timeouts_disabled_refcount) == 0; | ||
1654 | } | ||
1655 | |||
1656 | static inline u32 gk20a_get_gr_idle_timeout(struct gk20a *g) | ||
1657 | { | ||
1658 | return nvgpu_is_timeouts_enabled(g) ? | ||
1659 | g->gr_idle_timeout_default : UINT_MAX; | ||
1660 | } | ||
1661 | |||
1662 | #define MULTICHAR_TAG(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d)) | ||
1663 | enum BAR0_DEBUG_OPERATION { | ||
1664 | BARO_ZERO_NOP = 0, | ||
1665 | OP_END = MULTICHAR_TAG('D', 'O', 'N', 'E'), | ||
1666 | BAR0_READ32 = MULTICHAR_TAG('0', 'R', '3', '2'), | ||
1667 | BAR0_WRITE32 = MULTICHAR_TAG('0', 'W', '3', '2'), | ||
1668 | }; | ||
1669 | |||
1670 | struct share_buffer_head { | ||
1671 | enum BAR0_DEBUG_OPERATION operation; | ||
1672 | /* size of the operation item */ | ||
1673 | u32 size; | ||
1674 | u32 completed; | ||
1675 | u32 failed; | ||
1676 | u64 context; | ||
1677 | u64 completion_callback; | ||
1678 | }; | ||
1679 | |||
1680 | struct gk20a_cyclestate_buffer_elem { | ||
1681 | struct share_buffer_head head; | ||
1682 | /* in */ | ||
1683 | u64 p_data; | ||
1684 | u64 p_done; | ||
1685 | u32 offset_bar0; | ||
1686 | u16 first_bit; | ||
1687 | u16 last_bit; | ||
1688 | /* out */ | ||
1689 | /* keep 64 bits to be consistent */ | ||
1690 | u64 data; | ||
1691 | }; | ||
1692 | |||
1693 | /* operations that will need to be executed on non stall workqueue */ | ||
1694 | #define GK20A_NONSTALL_OPS_WAKEUP_SEMAPHORE BIT32(0) | ||
1695 | #define GK20A_NONSTALL_OPS_POST_EVENTS BIT32(1) | ||
1696 | |||
1697 | /* register accessors */ | ||
1698 | void __nvgpu_check_gpu_state(struct gk20a *g); | ||
1699 | void __gk20a_warn_on_no_regs(void); | ||
1700 | |||
1701 | /* classes that the device supports */ | ||
1702 | /* TBD: get these from an open-sourced SDK? */ | ||
1703 | enum { | ||
1704 | FERMI_TWOD_A = 0x902D, | ||
1705 | KEPLER_INLINE_TO_MEMORY_A = 0xA040, | ||
1706 | KEPLER_DMA_COPY_A = 0xA0B5, | ||
1707 | }; | ||
1708 | |||
1709 | #define GK20A_BAR0_IORESOURCE_MEM 0 | ||
1710 | #define GK20A_BAR1_IORESOURCE_MEM 1 | ||
1711 | #define GK20A_SIM_IORESOURCE_MEM 2 | ||
1712 | |||
1713 | void gk20a_busy_noresume(struct gk20a *g); | ||
1714 | void gk20a_idle_nosuspend(struct gk20a *g); | ||
1715 | int __must_check gk20a_busy(struct gk20a *g); | ||
1716 | void gk20a_idle(struct gk20a *g); | ||
1717 | int __gk20a_do_idle(struct gk20a *g, bool force_reset); | ||
1718 | int __gk20a_do_unidle(struct gk20a *g); | ||
1719 | |||
1720 | int gk20a_can_busy(struct gk20a *g); | ||
1721 | int gk20a_wait_for_idle(struct gk20a *g); | ||
1722 | |||
1723 | #define NVGPU_GPU_ARCHITECTURE_SHIFT 4 | ||
1724 | |||
1725 | /* constructs unique and compact GPUID from nvgpu_gpu_characteristics | ||
1726 | * arch/impl fields */ | ||
1727 | #define GK20A_GPUID(arch, impl) ((u32) ((arch) | (impl))) | ||
1728 | |||
1729 | #define GK20A_GPUID_GK20A 0x000000EA | ||
1730 | #define GK20A_GPUID_GM20B 0x0000012B | ||
1731 | #define GK20A_GPUID_GM20B_B 0x0000012E | ||
1732 | #define NVGPU_GPUID_GP10B 0x0000013B | ||
1733 | #define NVGPU_GPUID_GP104 0x00000134 | ||
1734 | #define NVGPU_GPUID_GP106 0x00000136 | ||
1735 | #define NVGPU_GPUID_GV11B 0x0000015B | ||
1736 | #define NVGPU_GPUID_GV100 0x00000140 | ||
1737 | |||
1738 | int gk20a_init_gpu_characteristics(struct gk20a *g); | ||
1739 | |||
1740 | int gk20a_prepare_poweroff(struct gk20a *g); | ||
1741 | int gk20a_finalize_poweron(struct gk20a *g); | ||
1742 | |||
1743 | void nvgpu_wait_for_deferred_interrupts(struct gk20a *g); | ||
1744 | 26 | ||
1745 | struct gk20a * __must_check gk20a_get(struct gk20a *g); | 27 | #ifndef GK20A_GK20A_H |
1746 | void gk20a_put(struct gk20a *g); | 28 | #define GK20A_GK20A_H |
1747 | 29 | ||
1748 | static inline bool gk20a_platform_has_syncpoints(struct gk20a *g) | 30 | /* no new headers should be added here */ |
1749 | { | 31 | #include <nvgpu/gk20a.h> |
1750 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
1751 | return g->has_syncpoints && !g->disable_syncpoints; | ||
1752 | #else | ||
1753 | return false; | ||
1754 | #endif | ||
1755 | } | ||
1756 | 32 | ||
1757 | int gk20a_detect_chip(struct gk20a *g); | 33 | #endif \ No newline at end of file |
1758 | #endif /* GK20A_H */ | ||
diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h new file mode 100644 index 00000000..31ca1b45 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h | |||
@@ -0,0 +1,1758 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * GK20A Graphics | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | #ifndef GK20A_H | ||
25 | #define GK20A_H | ||
26 | |||
27 | struct gk20a; | ||
28 | struct fifo_gk20a; | ||
29 | struct channel_gk20a; | ||
30 | struct gr_gk20a; | ||
31 | struct sim_nvgpu; | ||
32 | struct gk20a_ctxsw_ucode_segments; | ||
33 | struct gk20a_fecs_trace; | ||
34 | struct gk20a_ctxsw_trace; | ||
35 | struct acr_desc; | ||
36 | struct nvgpu_mem_alloc_tracker; | ||
37 | struct dbg_profiler_object_data; | ||
38 | struct gk20a_debug_output; | ||
39 | struct nvgpu_clk_pll_debug_data; | ||
40 | struct nvgpu_nvhost_dev; | ||
41 | struct nvgpu_cpu_time_correlation_sample; | ||
42 | struct nvgpu_mem_sgt; | ||
43 | struct nvgpu_warpstate; | ||
44 | struct nvgpu_clk_session; | ||
45 | struct nvgpu_clk_arb; | ||
46 | #ifdef CONFIG_GK20A_CTXSW_TRACE | ||
47 | struct nvgpu_gpu_ctxsw_trace_filter; | ||
48 | #endif | ||
49 | struct priv_cmd_entry; | ||
50 | struct nvgpu_gpfifo_args; | ||
51 | |||
52 | #include <nvgpu/lock.h> | ||
53 | #include <nvgpu/thread.h> | ||
54 | |||
55 | #include <nvgpu/mm.h> | ||
56 | #include <nvgpu/as.h> | ||
57 | #include <nvgpu/log.h> | ||
58 | #include <nvgpu/pramin.h> | ||
59 | #include <nvgpu/acr/nvgpu_acr.h> | ||
60 | #include <nvgpu/kref.h> | ||
61 | #include <nvgpu/falcon.h> | ||
62 | #include <nvgpu/pmu.h> | ||
63 | #include <nvgpu/atomic.h> | ||
64 | #include <nvgpu/barrier.h> | ||
65 | #include <nvgpu/rwsem.h> | ||
66 | #include <nvgpu/nvlink.h> | ||
67 | #include <nvgpu/sim.h> | ||
68 | #include <nvgpu/ecc.h> | ||
69 | |||
70 | #include "gk20a/clk_gk20a.h" | ||
71 | #include "gk20a/ce2_gk20a.h" | ||
72 | #include "gk20a/fifo_gk20a.h" | ||
73 | #include "gk20a/tsg_gk20a.h" | ||
74 | #include "clk/clk.h" | ||
75 | #include "perf/perf.h" | ||
76 | #include "pmgr/pmgr.h" | ||
77 | #include "therm/thrm.h" | ||
78 | |||
79 | #ifdef CONFIG_DEBUG_FS | ||
80 | struct railgate_stats { | ||
81 | unsigned long last_rail_gate_start; | ||
82 | unsigned long last_rail_gate_complete; | ||
83 | unsigned long last_rail_ungate_start; | ||
84 | unsigned long last_rail_ungate_complete; | ||
85 | unsigned long total_rail_gate_time_ms; | ||
86 | unsigned long total_rail_ungate_time_ms; | ||
87 | unsigned long railgating_cycle_count; | ||
88 | }; | ||
89 | #endif | ||
90 | |||
91 | enum gk20a_cbc_op { | ||
92 | gk20a_cbc_op_clear, | ||
93 | gk20a_cbc_op_clean, | ||
94 | gk20a_cbc_op_invalidate, | ||
95 | }; | ||
96 | |||
97 | #define MC_INTR_UNIT_DISABLE false | ||
98 | #define MC_INTR_UNIT_ENABLE true | ||
99 | |||
100 | #define GPU_LIT_NUM_GPCS 0 | ||
101 | #define GPU_LIT_NUM_PES_PER_GPC 1 | ||
102 | #define GPU_LIT_NUM_ZCULL_BANKS 2 | ||
103 | #define GPU_LIT_NUM_TPC_PER_GPC 3 | ||
104 | #define GPU_LIT_NUM_SM_PER_TPC 4 | ||
105 | #define GPU_LIT_NUM_FBPS 5 | ||
106 | #define GPU_LIT_GPC_BASE 6 | ||
107 | #define GPU_LIT_GPC_STRIDE 7 | ||
108 | #define GPU_LIT_GPC_SHARED_BASE 8 | ||
109 | #define GPU_LIT_TPC_IN_GPC_BASE 9 | ||
110 | #define GPU_LIT_TPC_IN_GPC_STRIDE 10 | ||
111 | #define GPU_LIT_TPC_IN_GPC_SHARED_BASE 11 | ||
112 | #define GPU_LIT_PPC_IN_GPC_BASE 12 | ||
113 | #define GPU_LIT_PPC_IN_GPC_STRIDE 13 | ||
114 | #define GPU_LIT_PPC_IN_GPC_SHARED_BASE 14 | ||
115 | #define GPU_LIT_ROP_BASE 15 | ||
116 | #define GPU_LIT_ROP_STRIDE 16 | ||
117 | #define GPU_LIT_ROP_SHARED_BASE 17 | ||
118 | #define GPU_LIT_HOST_NUM_ENGINES 18 | ||
119 | #define GPU_LIT_HOST_NUM_PBDMA 19 | ||
120 | #define GPU_LIT_LTC_STRIDE 20 | ||
121 | #define GPU_LIT_LTS_STRIDE 21 | ||
122 | #define GPU_LIT_NUM_FBPAS 22 | ||
123 | #define GPU_LIT_FBPA_STRIDE 23 | ||
124 | #define GPU_LIT_FBPA_BASE 24 | ||
125 | #define GPU_LIT_FBPA_SHARED_BASE 25 | ||
126 | #define GPU_LIT_SM_PRI_STRIDE 26 | ||
127 | #define GPU_LIT_SMPC_PRI_BASE 27 | ||
128 | #define GPU_LIT_SMPC_PRI_SHARED_BASE 28 | ||
129 | #define GPU_LIT_SMPC_PRI_UNIQUE_BASE 29 | ||
130 | #define GPU_LIT_SMPC_PRI_STRIDE 30 | ||
131 | #define GPU_LIT_TWOD_CLASS 31 | ||
132 | #define GPU_LIT_THREED_CLASS 32 | ||
133 | #define GPU_LIT_COMPUTE_CLASS 33 | ||
134 | #define GPU_LIT_GPFIFO_CLASS 34 | ||
135 | #define GPU_LIT_I2M_CLASS 35 | ||
136 | #define GPU_LIT_DMA_COPY_CLASS 36 | ||
137 | #define GPU_LIT_GPC_PRIV_STRIDE 37 | ||
138 | |||
139 | #define nvgpu_get_litter_value(g, v) (g)->ops.get_litter_value((g), v) | ||
140 | |||
141 | enum nvgpu_unit; | ||
142 | |||
143 | enum nvgpu_flush_op; | ||
144 | enum gk20a_mem_rw_flag; | ||
145 | |||
146 | struct _resmgr_context; | ||
147 | struct nvgpu_gpfifo_entry; | ||
148 | |||
149 | struct nvgpu_gpfifo_userdata { | ||
150 | struct nvgpu_gpfifo_entry __user *entries; | ||
151 | struct _resmgr_context *context; | ||
152 | }; | ||
153 | |||
154 | /* | ||
155 | * gpu_ops should only contain function pointers! Non-function pointer members | ||
156 | * should go in struct gk20a or be implemented with the boolean flag API defined | ||
157 | * in nvgpu/enabled.h | ||
158 | */ | ||
159 | |||
160 | /* index for FB fault buffer functions */ | ||
161 | #define NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX 0U | ||
162 | #define NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX 1U | ||
163 | #define NVGPU_FB_MMU_FAULT_BUF_DISABLED 0U | ||
164 | #define NVGPU_FB_MMU_FAULT_BUF_ENABLED 1U | ||
165 | |||
166 | /* Parameters for init_elcg_mode/init_blcg_mode */ | ||
167 | enum { | ||
168 | ELCG_RUN, /* clk always run, i.e. disable elcg */ | ||
169 | ELCG_STOP, /* clk is stopped */ | ||
170 | ELCG_AUTO /* clk will run when non-idle, standard elcg mode */ | ||
171 | }; | ||
172 | |||
173 | enum { | ||
174 | BLCG_RUN, /* clk always run, i.e. disable blcg */ | ||
175 | BLCG_AUTO /* clk will run when non-idle, standard blcg mode */ | ||
176 | }; | ||
177 | |||
178 | struct gpu_ops { | ||
179 | struct { | ||
180 | int (*determine_L2_size_bytes)(struct gk20a *gk20a); | ||
181 | u64 (*get_cbc_base_divisor)(struct gk20a *g); | ||
182 | int (*init_comptags)(struct gk20a *g, struct gr_gk20a *gr); | ||
183 | int (*cbc_ctrl)(struct gk20a *g, enum gk20a_cbc_op op, | ||
184 | u32 min, u32 max); | ||
185 | void (*set_zbc_color_entry)(struct gk20a *g, | ||
186 | struct zbc_entry *color_val, | ||
187 | u32 index); | ||
188 | void (*set_zbc_depth_entry)(struct gk20a *g, | ||
189 | struct zbc_entry *depth_val, | ||
190 | u32 index); | ||
191 | void (*set_zbc_s_entry)(struct gk20a *g, | ||
192 | struct zbc_entry *s_val, | ||
193 | u32 index); | ||
194 | void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr); | ||
195 | void (*set_enabled)(struct gk20a *g, bool enabled); | ||
196 | void (*init_fs_state)(struct gk20a *g); | ||
197 | void (*isr)(struct gk20a *g); | ||
198 | u32 (*cbc_fix_config)(struct gk20a *g, int base); | ||
199 | void (*flush)(struct gk20a *g); | ||
200 | void (*intr_en_illegal_compstat)(struct gk20a *g, bool enable); | ||
201 | bool (*pri_is_ltc_addr)(struct gk20a *g, u32 addr); | ||
202 | bool (*is_ltcs_ltss_addr)(struct gk20a *g, u32 addr); | ||
203 | bool (*is_ltcn_ltss_addr)(struct gk20a *g, u32 addr); | ||
204 | void (*split_lts_broadcast_addr)(struct gk20a *g, u32 addr, | ||
205 | u32 *priv_addr_table, | ||
206 | u32 *priv_addr_table_index); | ||
207 | void (*split_ltc_broadcast_addr)(struct gk20a *g, u32 addr, | ||
208 | u32 *priv_addr_table, | ||
209 | u32 *priv_addr_table_index); | ||
210 | } ltc; | ||
211 | struct { | ||
212 | void (*isr_stall)(struct gk20a *g, u32 inst_id, u32 pri_base); | ||
213 | u32 (*isr_nonstall)(struct gk20a *g, u32 inst_id, u32 pri_base); | ||
214 | u32 (*get_num_pce)(struct gk20a *g); | ||
215 | } ce2; | ||
216 | struct { | ||
217 | u32 (*get_patch_slots)(struct gk20a *g); | ||
218 | int (*init_fs_state)(struct gk20a *g); | ||
219 | int (*init_preemption_state)(struct gk20a *g); | ||
220 | void (*access_smpc_reg)(struct gk20a *g, u32 quad, u32 offset); | ||
221 | void (*bundle_cb_defaults)(struct gk20a *g); | ||
222 | void (*cb_size_default)(struct gk20a *g); | ||
223 | int (*calc_global_ctx_buffer_size)(struct gk20a *g); | ||
224 | void (*commit_global_attrib_cb)(struct gk20a *g, | ||
225 | struct nvgpu_gr_ctx *ch_ctx, | ||
226 | u64 addr, bool patch); | ||
227 | void (*commit_global_bundle_cb)(struct gk20a *g, | ||
228 | struct nvgpu_gr_ctx *ch_ctx, | ||
229 | u64 addr, u64 size, bool patch); | ||
230 | int (*commit_global_cb_manager)(struct gk20a *g, | ||
231 | struct channel_gk20a *ch, | ||
232 | bool patch); | ||
233 | void (*commit_global_pagepool)(struct gk20a *g, | ||
234 | struct nvgpu_gr_ctx *ch_ctx, | ||
235 | u64 addr, u32 size, bool patch); | ||
236 | void (*init_gpc_mmu)(struct gk20a *g); | ||
237 | int (*handle_sw_method)(struct gk20a *g, u32 addr, | ||
238 | u32 class_num, u32 offset, u32 data); | ||
239 | void (*set_alpha_circular_buffer_size)(struct gk20a *g, | ||
240 | u32 data); | ||
241 | void (*set_circular_buffer_size)(struct gk20a *g, u32 data); | ||
242 | void (*set_bes_crop_debug3)(struct gk20a *g, u32 data); | ||
243 | void (*set_bes_crop_debug4)(struct gk20a *g, u32 data); | ||
244 | void (*enable_hww_exceptions)(struct gk20a *g); | ||
245 | bool (*is_valid_class)(struct gk20a *g, u32 class_num); | ||
246 | bool (*is_valid_gfx_class)(struct gk20a *g, u32 class_num); | ||
247 | bool (*is_valid_compute_class)(struct gk20a *g, u32 class_num); | ||
248 | void (*get_sm_dsm_perf_regs)(struct gk20a *g, | ||
249 | u32 *num_sm_dsm_perf_regs, | ||
250 | u32 **sm_dsm_perf_regs, | ||
251 | u32 *perf_register_stride); | ||
252 | void (*get_sm_dsm_perf_ctrl_regs)(struct gk20a *g, | ||
253 | u32 *num_sm_dsm_perf_regs, | ||
254 | u32 **sm_dsm_perf_regs, | ||
255 | u32 *perf_register_stride); | ||
256 | void (*get_ovr_perf_regs)(struct gk20a *g, | ||
257 | u32 *num_ovr_perf_regs, | ||
258 | u32 **ovr_perf_regsr); | ||
259 | void (*set_hww_esr_report_mask)(struct gk20a *g); | ||
260 | int (*setup_alpha_beta_tables)(struct gk20a *g, | ||
261 | struct gr_gk20a *gr); | ||
262 | int (*falcon_load_ucode)(struct gk20a *g, | ||
263 | u64 addr_base, | ||
264 | struct gk20a_ctxsw_ucode_segments *segments, | ||
265 | u32 reg_offset); | ||
266 | int (*load_ctxsw_ucode)(struct gk20a *g); | ||
267 | u32 (*get_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index); | ||
268 | void (*set_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index); | ||
269 | int (*alloc_obj_ctx)(struct channel_gk20a *c, | ||
270 | u32 class_num, u32 flags); | ||
271 | int (*bind_ctxsw_zcull)(struct gk20a *g, struct gr_gk20a *gr, | ||
272 | struct channel_gk20a *c, u64 zcull_va, | ||
273 | u32 mode); | ||
274 | int (*get_zcull_info)(struct gk20a *g, struct gr_gk20a *gr, | ||
275 | struct gr_zcull_info *zcull_params); | ||
276 | int (*decode_egpc_addr)(struct gk20a *g, | ||
277 | u32 addr, enum ctxsw_addr_type *addr_type, | ||
278 | u32 *gpc_num, u32 *tpc_num, u32 *broadcast_flags); | ||
279 | void (*egpc_etpc_priv_addr_table)(struct gk20a *g, u32 addr, | ||
280 | u32 gpc, u32 tpc, u32 broadcast_flags, | ||
281 | u32 *priv_addr_table, | ||
282 | u32 *priv_addr_table_index); | ||
283 | bool (*is_tpc_addr)(struct gk20a *g, u32 addr); | ||
284 | bool (*is_egpc_addr)(struct gk20a *g, u32 addr); | ||
285 | bool (*is_etpc_addr)(struct gk20a *g, u32 addr); | ||
286 | void (*get_egpc_etpc_num)(struct gk20a *g, u32 addr, | ||
287 | u32 *gpc_num, u32 *tpc_num); | ||
288 | u32 (*get_tpc_num)(struct gk20a *g, u32 addr); | ||
289 | u32 (*get_egpc_base)(struct gk20a *g); | ||
290 | void (*detect_sm_arch)(struct gk20a *g); | ||
291 | int (*add_zbc_color)(struct gk20a *g, struct gr_gk20a *gr, | ||
292 | struct zbc_entry *color_val, u32 index); | ||
293 | int (*add_zbc_depth)(struct gk20a *g, struct gr_gk20a *gr, | ||
294 | struct zbc_entry *depth_val, u32 index); | ||
295 | int (*add_zbc_s)(struct gk20a *g, struct gr_gk20a *gr, | ||
296 | struct zbc_entry *s_val, u32 index); | ||
297 | int (*zbc_set_table)(struct gk20a *g, struct gr_gk20a *gr, | ||
298 | struct zbc_entry *zbc_val); | ||
299 | int (*zbc_query_table)(struct gk20a *g, struct gr_gk20a *gr, | ||
300 | struct zbc_query_params *query_params); | ||
301 | int (*zbc_s_query_table)(struct gk20a *g, struct gr_gk20a *gr, | ||
302 | struct zbc_query_params *query_params); | ||
303 | int (*load_zbc_s_default_tbl)(struct gk20a *g, | ||
304 | struct gr_gk20a *gr); | ||
305 | int (*load_zbc_s_tbl)(struct gk20a *g, | ||
306 | struct gr_gk20a *gr); | ||
307 | void (*pmu_save_zbc)(struct gk20a *g, u32 entries); | ||
308 | int (*add_zbc)(struct gk20a *g, struct gr_gk20a *gr, | ||
309 | struct zbc_entry *zbc_val); | ||
310 | bool (*add_zbc_type_s)(struct gk20a *g, struct gr_gk20a *gr, | ||
311 | struct zbc_entry *zbc_val, int *ret_val); | ||
312 | u32 (*pagepool_default_size)(struct gk20a *g); | ||
313 | int (*init_ctx_state)(struct gk20a *g); | ||
314 | int (*alloc_gr_ctx)(struct gk20a *g, | ||
315 | struct nvgpu_gr_ctx *gr_ctx, struct vm_gk20a *vm, | ||
316 | u32 class, u32 padding); | ||
317 | void (*free_gr_ctx)(struct gk20a *g, | ||
318 | struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx); | ||
319 | void (*powergate_tpc)(struct gk20a *g); | ||
320 | void (*update_ctxsw_preemption_mode)(struct gk20a *g, | ||
321 | struct channel_gk20a *c, | ||
322 | struct nvgpu_mem *mem); | ||
323 | int (*update_smpc_ctxsw_mode)(struct gk20a *g, | ||
324 | struct channel_gk20a *c, | ||
325 | bool enable); | ||
326 | u32 (*get_hw_accessor_stream_out_mode)(void); | ||
327 | int (*update_hwpm_ctxsw_mode)(struct gk20a *g, | ||
328 | struct channel_gk20a *c, | ||
329 | u64 gpu_va, | ||
330 | u32 mode); | ||
331 | void (*init_hwpm_pmm_register)(struct gk20a *g); | ||
332 | int (*dump_gr_regs)(struct gk20a *g, | ||
333 | struct gk20a_debug_output *o); | ||
334 | int (*update_pc_sampling)(struct channel_gk20a *ch, | ||
335 | bool enable); | ||
336 | u32 (*get_max_fbps_count)(struct gk20a *g); | ||
337 | u32 (*get_fbp_en_mask)(struct gk20a *g); | ||
338 | u32 (*get_max_ltc_per_fbp)(struct gk20a *g); | ||
339 | u32 (*get_max_lts_per_ltc)(struct gk20a *g); | ||
340 | u32* (*get_rop_l2_en_mask)(struct gk20a *g); | ||
341 | void (*init_sm_dsm_reg_info)(void); | ||
342 | void (*init_ovr_sm_dsm_perf)(void); | ||
343 | int (*wait_empty)(struct gk20a *g, unsigned long duration_ms, | ||
344 | u32 expect_delay); | ||
345 | void (*init_cyclestats)(struct gk20a *g); | ||
346 | void (*enable_cde_in_fecs)(struct gk20a *g, | ||
347 | struct nvgpu_mem *mem); | ||
348 | int (*set_sm_debug_mode)(struct gk20a *g, struct channel_gk20a *ch, | ||
349 | u64 sms, bool enable); | ||
350 | void (*bpt_reg_info)(struct gk20a *g, | ||
351 | struct nvgpu_warpstate *w_state); | ||
352 | void (*get_access_map)(struct gk20a *g, | ||
353 | u32 **whitelist, int *num_entries); | ||
354 | int (*handle_fecs_error)(struct gk20a *g, | ||
355 | struct channel_gk20a *ch, | ||
356 | struct gr_gk20a_isr_data *isr_data); | ||
357 | int (*pre_process_sm_exception)(struct gk20a *g, | ||
358 | u32 gpc, u32 tpc, u32 sm, u32 global_esr, u32 warp_esr, | ||
359 | bool sm_debugger_attached, | ||
360 | struct channel_gk20a *fault_ch, | ||
361 | bool *early_exit, bool *ignore_debugger); | ||
362 | u32 (*get_sm_hww_warp_esr)(struct gk20a *g, | ||
363 | u32 gpc, u32 tpc, u32 sm); | ||
364 | u32 (*get_sm_hww_global_esr)(struct gk20a *g, | ||
365 | u32 gpc, u32 tpc, u32 sm); | ||
366 | u32 (*get_sm_no_lock_down_hww_global_esr_mask)(struct gk20a *g); | ||
367 | int (*lock_down_sm)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, | ||
368 | u32 global_esr_mask, bool check_errors); | ||
369 | int (*wait_for_sm_lock_down)(struct gk20a *g, u32 gpc, u32 tpc, | ||
370 | u32 sm, u32 global_esr_mask, bool check_errors); | ||
371 | void (*clear_sm_hww)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, | ||
372 | u32 global_esr); | ||
373 | void (*get_esr_sm_sel)(struct gk20a *g, u32 gpc, u32 tpc, | ||
374 | u32 *esr_sm_sel); | ||
375 | int (*handle_tpc_sm_ecc_exception)(struct gk20a *g, | ||
376 | u32 gpc, u32 tpc, | ||
377 | bool *post_event, struct channel_gk20a *fault_ch, | ||
378 | u32 *hww_global_esr); | ||
379 | int (*handle_sm_exception)(struct gk20a *g, | ||
380 | u32 gpc, u32 tpc, u32 sm, | ||
381 | bool *post_event, struct channel_gk20a *fault_ch, | ||
382 | u32 *hww_global_esr); | ||
383 | int (*handle_gcc_exception)(struct gk20a *g, u32 gpc, u32 tpc, | ||
384 | bool *post_event, struct channel_gk20a *fault_ch, | ||
385 | u32 *hww_global_esr); | ||
386 | int (*handle_tex_exception)(struct gk20a *g, u32 gpc, u32 tpc, | ||
387 | bool *post_event); | ||
388 | int (*handle_tpc_mpc_exception)(struct gk20a *g, | ||
389 | u32 gpc, u32 tpc, bool *post_event); | ||
390 | int (*handle_gpc_gpccs_exception)(struct gk20a *g, u32 gpc, | ||
391 | u32 gpc_exception); | ||
392 | int (*handle_gpc_gpcmmu_exception)(struct gk20a *g, u32 gpc, | ||
393 | u32 gpc_exception); | ||
394 | void (*enable_gpc_exceptions)(struct gk20a *g); | ||
395 | void (*enable_exceptions)(struct gk20a *g); | ||
396 | int (*init_ecc)(struct gk20a *g); | ||
397 | u32 (*get_lrf_tex_ltc_dram_override)(struct gk20a *g); | ||
398 | int (*record_sm_error_state)(struct gk20a *g, u32 gpc, u32 tpc, | ||
399 | u32 sm, struct channel_gk20a *fault_ch); | ||
400 | int (*update_sm_error_state)(struct gk20a *g, | ||
401 | struct channel_gk20a *ch, u32 sm_id, | ||
402 | struct nvgpu_tsg_sm_error_state *sm_error_state); | ||
403 | int (*clear_sm_error_state)(struct gk20a *g, | ||
404 | struct channel_gk20a *ch, u32 sm_id); | ||
405 | int (*suspend_contexts)(struct gk20a *g, | ||
406 | struct dbg_session_gk20a *dbg_s, | ||
407 | int *ctx_resident_ch_fd); | ||
408 | int (*resume_contexts)(struct gk20a *g, | ||
409 | struct dbg_session_gk20a *dbg_s, | ||
410 | int *ctx_resident_ch_fd); | ||
411 | int (*set_preemption_mode)(struct channel_gk20a *ch, | ||
412 | u32 graphics_preempt_mode, | ||
413 | u32 compute_preempt_mode); | ||
414 | int (*get_preemption_mode_flags)(struct gk20a *g, | ||
415 | struct nvgpu_preemption_modes_rec *preemption_modes_rec); | ||
416 | int (*set_ctxsw_preemption_mode)(struct gk20a *g, | ||
417 | struct nvgpu_gr_ctx *gr_ctx, | ||
418 | struct vm_gk20a *vm, u32 class, | ||
419 | u32 graphics_preempt_mode, | ||
420 | u32 compute_preempt_mode); | ||
421 | int (*set_boosted_ctx)(struct channel_gk20a *ch, bool boost); | ||
422 | void (*update_boosted_ctx)(struct gk20a *g, | ||
423 | struct nvgpu_mem *mem, | ||
424 | struct nvgpu_gr_ctx *gr_ctx); | ||
425 | int (*init_sm_id_table)(struct gk20a *g); | ||
426 | int (*load_smid_config)(struct gk20a *g); | ||
427 | void (*program_sm_id_numbering)(struct gk20a *g, | ||
428 | u32 gpc, u32 tpc, u32 smid); | ||
429 | void (*program_active_tpc_counts)(struct gk20a *g, u32 gpc); | ||
430 | int (*setup_rop_mapping)(struct gk20a *g, struct gr_gk20a *gr); | ||
431 | int (*init_sw_veid_bundle)(struct gk20a *g); | ||
432 | void (*program_zcull_mapping)(struct gk20a *g, | ||
433 | u32 zcull_alloc_num, u32 *zcull_map_tiles); | ||
434 | int (*commit_global_timeslice)(struct gk20a *g, | ||
435 | struct channel_gk20a *c); | ||
436 | int (*commit_inst)(struct channel_gk20a *c, u64 gpu_va); | ||
437 | void (*write_zcull_ptr)(struct gk20a *g, | ||
438 | struct nvgpu_mem *mem, u64 gpu_va); | ||
439 | void (*write_pm_ptr)(struct gk20a *g, | ||
440 | struct nvgpu_mem *mem, u64 gpu_va); | ||
441 | void (*set_preemption_buffer_va)(struct gk20a *g, | ||
442 | struct nvgpu_mem *mem, u64 gpu_va); | ||
443 | void (*load_tpc_mask)(struct gk20a *g); | ||
444 | int (*trigger_suspend)(struct gk20a *g); | ||
445 | int (*wait_for_pause)(struct gk20a *g, struct nvgpu_warpstate *w_state); | ||
446 | int (*resume_from_pause)(struct gk20a *g); | ||
447 | int (*clear_sm_errors)(struct gk20a *g); | ||
448 | u32 (*tpc_enabled_exceptions)(struct gk20a *g); | ||
449 | int (*set_czf_bypass)(struct gk20a *g, | ||
450 | struct channel_gk20a *ch); | ||
451 | void (*init_czf_bypass)(struct gk20a *g); | ||
452 | bool (*sm_debugger_attached)(struct gk20a *g); | ||
453 | void (*suspend_single_sm)(struct gk20a *g, | ||
454 | u32 gpc, u32 tpc, u32 sm, | ||
455 | u32 global_esr_mask, bool check_errors); | ||
456 | void (*suspend_all_sms)(struct gk20a *g, | ||
457 | u32 global_esr_mask, bool check_errors); | ||
458 | void (*resume_single_sm)(struct gk20a *g, | ||
459 | u32 gpc, u32 tpc, u32 sm); | ||
460 | void (*resume_all_sms)(struct gk20a *g); | ||
461 | void (*disable_rd_coalesce)(struct gk20a *g); | ||
462 | void (*init_ctxsw_hdr_data)(struct gk20a *g, | ||
463 | struct nvgpu_mem *mem); | ||
464 | void (*init_gfxp_wfi_timeout_count)(struct gk20a *g); | ||
465 | unsigned long (*get_max_gfxp_wfi_timeout_count) | ||
466 | (struct gk20a *g); | ||
467 | void (*ecc_init_scrub_reg)(struct gk20a *g); | ||
468 | u32 (*get_gpcs_swdx_dss_zbc_c_format_reg)(struct gk20a *g); | ||
469 | u32 (*get_gpcs_swdx_dss_zbc_z_format_reg)(struct gk20a *g); | ||
470 | void (*dump_ctxsw_stats)(struct gk20a *g, struct vm_gk20a *vm, | ||
471 | struct nvgpu_gr_ctx *gr_ctx); | ||
472 | void (*fecs_host_int_enable)(struct gk20a *g); | ||
473 | int (*handle_ssync_hww)(struct gk20a *g); | ||
474 | int (*handle_notify_pending)(struct gk20a *g, | ||
475 | struct gr_gk20a_isr_data *isr_data); | ||
476 | int (*handle_semaphore_pending)(struct gk20a *g, | ||
477 | struct gr_gk20a_isr_data *isr_data); | ||
478 | int (*add_ctxsw_reg_pm_fbpa)(struct gk20a *g, | ||
479 | struct ctxsw_buf_offset_map_entry *map, | ||
480 | struct aiv_list_gk20a *regs, | ||
481 | u32 *count, u32 *offset, | ||
482 | u32 max_cnt, u32 base, | ||
483 | u32 num_fbpas, u32 stride, u32 mask); | ||
484 | int (*add_ctxsw_reg_perf_pma)(struct ctxsw_buf_offset_map_entry *map, | ||
485 | struct aiv_list_gk20a *regs, | ||
486 | u32 *count, u32 *offset, | ||
487 | u32 max_cnt, u32 base, u32 mask); | ||
488 | int (*decode_priv_addr)(struct gk20a *g, u32 addr, | ||
489 | enum ctxsw_addr_type *addr_type, | ||
490 | u32 *gpc_num, u32 *tpc_num, | ||
491 | u32 *ppc_num, u32 *be_num, | ||
492 | u32 *broadcast_flags); | ||
493 | int (*create_priv_addr_table)(struct gk20a *g, | ||
494 | u32 addr, | ||
495 | u32 *priv_addr_table, | ||
496 | u32 *num_registers); | ||
497 | u32 (*get_pmm_per_chiplet_offset)(void); | ||
498 | void (*split_fbpa_broadcast_addr)(struct gk20a *g, u32 addr, | ||
499 | u32 num_fbpas, | ||
500 | u32 *priv_addr_table, | ||
501 | u32 *priv_addr_table_index); | ||
502 | u32 (*fecs_ctxsw_mailbox_size)(void); | ||
503 | int (*init_sw_bundle64)(struct gk20a *g); | ||
504 | int (*alloc_global_ctx_buffers)(struct gk20a *g); | ||
505 | int (*map_global_ctx_buffers)(struct gk20a *g, | ||
506 | struct channel_gk20a *c); | ||
507 | int (*commit_global_ctx_buffers)(struct gk20a *g, | ||
508 | struct channel_gk20a *c, bool patch); | ||
509 | u32 (*get_nonpes_aware_tpc)(struct gk20a *g, u32 gpc, u32 tpc); | ||
510 | int (*get_offset_in_gpccs_segment)(struct gk20a *g, | ||
511 | enum ctxsw_addr_type addr_type, u32 num_tpcs, | ||
512 | u32 num_ppcs, u32 reg_list_ppc_count, | ||
513 | u32 *__offset_in_segment); | ||
514 | void (*set_debug_mode)(struct gk20a *g, bool enable); | ||
515 | } gr; | ||
516 | struct { | ||
517 | void (*init_hw)(struct gk20a *g); | ||
518 | void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr); | ||
519 | void (*init_fs_state)(struct gk20a *g); | ||
520 | void (*reset)(struct gk20a *g); | ||
521 | void (*init_uncompressed_kind_map)(struct gk20a *g); | ||
522 | void (*init_kind_attr)(struct gk20a *g); | ||
523 | void (*set_mmu_page_size)(struct gk20a *g); | ||
524 | bool (*set_use_full_comp_tag_line)(struct gk20a *g); | ||
525 | u32 (*mmu_ctrl)(struct gk20a *g); | ||
526 | u32 (*mmu_debug_ctrl)(struct gk20a *g); | ||
527 | u32 (*mmu_debug_wr)(struct gk20a *g); | ||
528 | u32 (*mmu_debug_rd)(struct gk20a *g); | ||
529 | |||
530 | /* | ||
531 | * Compression tag line coverage. When mapping a compressible | ||
532 | * buffer, ctagline is increased when the virtual address | ||
533 | * crosses over the compression page boundary. | ||
534 | */ | ||
535 | unsigned int (*compression_page_size)(struct gk20a *g); | ||
536 | |||
537 | /* | ||
538 | * Minimum page size that can be used for compressible kinds. | ||
539 | */ | ||
540 | unsigned int (*compressible_page_size)(struct gk20a *g); | ||
541 | |||
542 | /* | ||
543 | * Compressible kind mappings: Mask for the virtual and physical | ||
544 | * address bits that must match. | ||
545 | */ | ||
546 | u32 (*compression_align_mask)(struct gk20a *g); | ||
547 | |||
548 | void (*dump_vpr_wpr_info)(struct gk20a *g); | ||
549 | int (*vpr_info_fetch)(struct gk20a *g); | ||
550 | void (*read_wpr_info)(struct gk20a *g, | ||
551 | struct wpr_carveout_info *inf); | ||
552 | bool (*is_debug_mode_enabled)(struct gk20a *g); | ||
553 | void (*set_debug_mode)(struct gk20a *g, bool enable); | ||
554 | int (*tlb_invalidate)(struct gk20a *g, struct nvgpu_mem *pdb); | ||
555 | void (*hub_isr)(struct gk20a *g); | ||
556 | void (*handle_replayable_fault)(struct gk20a *g); | ||
557 | int (*mem_unlock)(struct gk20a *g); | ||
558 | int (*init_nvlink)(struct gk20a *g); | ||
559 | int (*enable_nvlink)(struct gk20a *g); | ||
560 | void (*enable_hub_intr)(struct gk20a *g); | ||
561 | void (*disable_hub_intr)(struct gk20a *g); | ||
562 | int (*init_fbpa)(struct gk20a *g); | ||
563 | void (*fbpa_isr)(struct gk20a *g); | ||
564 | void (*write_mmu_fault_buffer_lo_hi)(struct gk20a *g, u32 index, | ||
565 | u32 addr_lo, u32 addr_hi); | ||
566 | void (*write_mmu_fault_buffer_get)(struct gk20a *g, u32 index, | ||
567 | u32 reg_val); | ||
568 | void (*write_mmu_fault_buffer_size)(struct gk20a *g, u32 index, | ||
569 | u32 reg_val); | ||
570 | void (*write_mmu_fault_status)(struct gk20a *g, u32 reg_val); | ||
571 | u32 (*read_mmu_fault_buffer_get)(struct gk20a *g, u32 index); | ||
572 | u32 (*read_mmu_fault_buffer_put)(struct gk20a *g, u32 index); | ||
573 | u32 (*read_mmu_fault_buffer_size)(struct gk20a *g, u32 index); | ||
574 | void (*read_mmu_fault_addr_lo_hi)(struct gk20a *g, | ||
575 | u32 *addr_lo, u32 *addr_hi); | ||
576 | void (*read_mmu_fault_inst_lo_hi)(struct gk20a *g, | ||
577 | u32 *inst_lo, u32 *inst_hi); | ||
578 | u32 (*read_mmu_fault_info)(struct gk20a *g); | ||
579 | u32 (*read_mmu_fault_status)(struct gk20a *g); | ||
580 | int (*mmu_invalidate_replay)(struct gk20a *g, | ||
581 | u32 invalidate_replay_val); | ||
582 | bool (*mmu_fault_pending)(struct gk20a *g); | ||
583 | bool (*is_fault_buf_enabled)(struct gk20a *g, u32 index); | ||
584 | void (*fault_buf_set_state_hw)(struct gk20a *g, | ||
585 | u32 index, u32 state); | ||
586 | void (*fault_buf_configure_hw)(struct gk20a *g, u32 index); | ||
587 | size_t (*get_vidmem_size)(struct gk20a *g); | ||
588 | } fb; | ||
589 | struct { | ||
590 | void (*slcg_bus_load_gating_prod)(struct gk20a *g, bool prod); | ||
591 | void (*slcg_ce2_load_gating_prod)(struct gk20a *g, bool prod); | ||
592 | void (*slcg_chiplet_load_gating_prod)(struct gk20a *g, bool prod); | ||
593 | void (*slcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod); | ||
594 | void (*slcg_fb_load_gating_prod)(struct gk20a *g, bool prod); | ||
595 | void (*slcg_fifo_load_gating_prod)(struct gk20a *g, bool prod); | ||
596 | void (*slcg_gr_load_gating_prod)(struct gk20a *g, bool prod); | ||
597 | void (*slcg_ltc_load_gating_prod)(struct gk20a *g, bool prod); | ||
598 | void (*slcg_perf_load_gating_prod)(struct gk20a *g, bool prod); | ||
599 | void (*slcg_priring_load_gating_prod)(struct gk20a *g, bool prod); | ||
600 | void (*slcg_pmu_load_gating_prod)(struct gk20a *g, bool prod); | ||
601 | void (*slcg_therm_load_gating_prod)(struct gk20a *g, bool prod); | ||
602 | void (*slcg_xbar_load_gating_prod)(struct gk20a *g, bool prod); | ||
603 | void (*blcg_bus_load_gating_prod)(struct gk20a *g, bool prod); | ||
604 | void (*blcg_ce_load_gating_prod)(struct gk20a *g, bool prod); | ||
605 | void (*blcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod); | ||
606 | void (*blcg_fb_load_gating_prod)(struct gk20a *g, bool prod); | ||
607 | void (*blcg_fifo_load_gating_prod)(struct gk20a *g, bool prod); | ||
608 | void (*blcg_gr_load_gating_prod)(struct gk20a *g, bool prod); | ||
609 | void (*blcg_ltc_load_gating_prod)(struct gk20a *g, bool prod); | ||
610 | void (*blcg_pwr_csb_load_gating_prod)(struct gk20a *g, bool prod); | ||
611 | void (*blcg_pmu_load_gating_prod)(struct gk20a *g, bool prod); | ||
612 | void (*blcg_xbar_load_gating_prod)(struct gk20a *g, bool prod); | ||
613 | void (*pg_gr_load_gating_prod)(struct gk20a *g, bool prod); | ||
614 | } clock_gating; | ||
615 | struct { | ||
616 | void (*post_events)(struct channel_gk20a *ch); | ||
617 | } debugger; | ||
618 | struct { | ||
619 | int (*setup_sw)(struct gk20a *g); | ||
620 | int (*init_fifo_setup_hw)(struct gk20a *g); | ||
621 | void (*bind_channel)(struct channel_gk20a *ch_gk20a); | ||
622 | void (*unbind_channel)(struct channel_gk20a *ch_gk20a); | ||
623 | void (*disable_channel)(struct channel_gk20a *ch); | ||
624 | void (*enable_channel)(struct channel_gk20a *ch); | ||
625 | int (*alloc_inst)(struct gk20a *g, struct channel_gk20a *ch); | ||
626 | void (*free_inst)(struct gk20a *g, struct channel_gk20a *ch); | ||
627 | int (*setup_ramfc)(struct channel_gk20a *c, u64 gpfifo_base, | ||
628 | u32 gpfifo_entries, | ||
629 | unsigned long acquire_timeout, | ||
630 | u32 flags); | ||
631 | int (*resetup_ramfc)(struct channel_gk20a *c); | ||
632 | int (*preempt_channel)(struct gk20a *g, u32 chid); | ||
633 | int (*preempt_tsg)(struct gk20a *g, u32 tsgid); | ||
634 | int (*enable_tsg)(struct tsg_gk20a *tsg); | ||
635 | int (*disable_tsg)(struct tsg_gk20a *tsg); | ||
636 | int (*tsg_verify_channel_status)(struct channel_gk20a *ch); | ||
637 | void (*tsg_verify_status_ctx_reload)(struct channel_gk20a *ch); | ||
638 | void (*tsg_verify_status_faulted)(struct channel_gk20a *ch); | ||
639 | int (*reschedule_runlist)(struct channel_gk20a *ch, | ||
640 | bool preempt_next); | ||
641 | int (*update_runlist)(struct gk20a *g, u32 runlist_id, | ||
642 | u32 chid, bool add, | ||
643 | bool wait_for_finish); | ||
644 | void (*trigger_mmu_fault)(struct gk20a *g, | ||
645 | unsigned long engine_ids); | ||
646 | void (*get_mmu_fault_info)(struct gk20a *g, u32 mmu_fault_id, | ||
647 | struct mmu_fault_info *mmfault); | ||
648 | void (*get_mmu_fault_desc)(struct mmu_fault_info *mmfault); | ||
649 | void (*get_mmu_fault_client_desc)( | ||
650 | struct mmu_fault_info *mmfault); | ||
651 | void (*get_mmu_fault_gpc_desc)(struct mmu_fault_info *mmfault); | ||
652 | void (*apply_pb_timeout)(struct gk20a *g); | ||
653 | void (*apply_ctxsw_timeout_intr)(struct gk20a *g); | ||
654 | int (*wait_engine_idle)(struct gk20a *g); | ||
655 | u32 (*get_num_fifos)(struct gk20a *g); | ||
656 | u32 (*get_pbdma_signature)(struct gk20a *g); | ||
657 | int (*set_runlist_interleave)(struct gk20a *g, u32 id, | ||
658 | u32 runlist_id, | ||
659 | u32 new_level); | ||
660 | int (*tsg_set_timeslice)(struct tsg_gk20a *tsg, u32 timeslice); | ||
661 | u32 (*default_timeslice_us)(struct gk20a *); | ||
662 | int (*force_reset_ch)(struct channel_gk20a *ch, | ||
663 | u32 err_code, bool verbose); | ||
664 | int (*engine_enum_from_type)(struct gk20a *g, u32 engine_type, | ||
665 | u32 *inst_id); | ||
666 | void (*device_info_data_parse)(struct gk20a *g, | ||
667 | u32 table_entry, u32 *inst_id, | ||
668 | u32 *pri_base, u32 *fault_id); | ||
669 | u32 (*device_info_fault_id)(u32 table_entry); | ||
670 | int (*tsg_bind_channel)(struct tsg_gk20a *tsg, | ||
671 | struct channel_gk20a *ch); | ||
672 | int (*tsg_unbind_channel)(struct channel_gk20a *ch); | ||
673 | int (*tsg_open)(struct tsg_gk20a *tsg); | ||
674 | void (*tsg_release)(struct tsg_gk20a *tsg); | ||
675 | u32 (*eng_runlist_base_size)(void); | ||
676 | int (*init_engine_info)(struct fifo_gk20a *f); | ||
677 | u32 (*runlist_entry_size)(void); | ||
678 | void (*get_tsg_runlist_entry)(struct tsg_gk20a *tsg, | ||
679 | u32 *runlist); | ||
680 | void (*get_ch_runlist_entry)(struct channel_gk20a *ch, | ||
681 | u32 *runlist); | ||
682 | u32 (*userd_gp_get)(struct gk20a *g, struct channel_gk20a *ch); | ||
683 | void (*userd_gp_put)(struct gk20a *g, struct channel_gk20a *ch); | ||
684 | u64 (*userd_pb_get)(struct gk20a *g, struct channel_gk20a *ch); | ||
685 | void (*free_channel_ctx_header)(struct channel_gk20a *ch); | ||
686 | bool (*is_fault_engine_subid_gpc)(struct gk20a *g, | ||
687 | u32 engine_subid); | ||
688 | void (*dump_pbdma_status)(struct gk20a *g, | ||
689 | struct gk20a_debug_output *o); | ||
690 | void (*dump_eng_status)(struct gk20a *g, | ||
691 | struct gk20a_debug_output *o); | ||
692 | void (*dump_channel_status_ramfc)(struct gk20a *g, | ||
693 | struct gk20a_debug_output *o, u32 chid, | ||
694 | struct ch_state *ch_state); | ||
695 | u32 (*intr_0_error_mask)(struct gk20a *g); | ||
696 | int (*is_preempt_pending)(struct gk20a *g, u32 id, | ||
697 | unsigned int id_type); | ||
698 | void (*init_pbdma_intr_descs)(struct fifo_gk20a *f); | ||
699 | int (*reset_enable_hw)(struct gk20a *g); | ||
700 | int (*setup_userd)(struct channel_gk20a *c); | ||
701 | u32 (*pbdma_acquire_val)(u64 timeout); | ||
702 | void (*teardown_ch_tsg)(struct gk20a *g, u32 act_eng_bitmask, | ||
703 | u32 id, unsigned int id_type, unsigned int rc_type, | ||
704 | struct mmu_fault_info *mmfault); | ||
705 | bool (*handle_sched_error)(struct gk20a *g); | ||
706 | bool (*handle_ctxsw_timeout)(struct gk20a *g, u32 fifo_intr); | ||
707 | unsigned int (*handle_pbdma_intr_0)(struct gk20a *g, | ||
708 | u32 pbdma_id, u32 pbdma_intr_0, | ||
709 | u32 *handled, u32 *error_notifier); | ||
710 | unsigned int (*handle_pbdma_intr_1)(struct gk20a *g, | ||
711 | u32 pbdma_id, u32 pbdma_intr_1, | ||
712 | u32 *handled, u32 *error_notifier); | ||
713 | void (*init_eng_method_buffers)(struct gk20a *g, | ||
714 | struct tsg_gk20a *tsg); | ||
715 | void (*deinit_eng_method_buffers)(struct gk20a *g, | ||
716 | struct tsg_gk20a *tsg); | ||
717 | u32 (*get_preempt_timeout)(struct gk20a *g); | ||
718 | void (*post_event_id)(struct tsg_gk20a *tsg, int event_id); | ||
719 | void (*ch_abort_clean_up)(struct channel_gk20a *ch); | ||
720 | bool (*check_tsg_ctxsw_timeout)(struct tsg_gk20a *tsg, | ||
721 | bool *verbose, u32 *ms); | ||
722 | bool (*check_ch_ctxsw_timeout)(struct channel_gk20a *ch, | ||
723 | bool *verbose, u32 *ms); | ||
724 | int (*channel_suspend)(struct gk20a *g); | ||
725 | int (*channel_resume)(struct gk20a *g); | ||
726 | void (*set_error_notifier)(struct channel_gk20a *ch, u32 error); | ||
727 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
728 | int (*alloc_syncpt_buf)(struct channel_gk20a *c, | ||
729 | u32 syncpt_id, struct nvgpu_mem *syncpt_buf); | ||
730 | void (*free_syncpt_buf)(struct channel_gk20a *c, | ||
731 | struct nvgpu_mem *syncpt_buf); | ||
732 | void (*add_syncpt_wait_cmd)(struct gk20a *g, | ||
733 | struct priv_cmd_entry *cmd, u32 off, | ||
734 | u32 id, u32 thresh, u64 gpu_va); | ||
735 | u32 (*get_syncpt_wait_cmd_size)(void); | ||
736 | void (*add_syncpt_incr_cmd)(struct gk20a *g, | ||
737 | bool wfi_cmd, struct priv_cmd_entry *cmd, | ||
738 | u32 id, u64 gpu_va); | ||
739 | u32 (*get_syncpt_incr_cmd_size)(bool wfi_cmd); | ||
740 | int (*get_sync_ro_map)(struct vm_gk20a *vm, | ||
741 | u64 *base_gpuva, u32 *sync_size); | ||
742 | u32 (*get_syncpt_incr_per_release)(void); | ||
743 | #endif | ||
744 | void (*runlist_hw_submit)(struct gk20a *g, u32 runlist_id, | ||
745 | u32 count, u32 buffer_index); | ||
746 | int (*runlist_wait_pending)(struct gk20a *g, u32 runlist_id); | ||
747 | void (*ring_channel_doorbell)(struct channel_gk20a *c); | ||
748 | u32 (*get_sema_wait_cmd_size)(void); | ||
749 | u32 (*get_sema_incr_cmd_size)(void); | ||
750 | void (*add_sema_cmd)(struct gk20a *g, | ||
751 | struct nvgpu_semaphore *s, u64 sema_va, | ||
752 | struct priv_cmd_entry *cmd, | ||
753 | u32 off, bool acquire, bool wfi); | ||
754 | } fifo; | ||
755 | struct pmu_v { | ||
756 | u32 (*get_pmu_cmdline_args_size)(struct nvgpu_pmu *pmu); | ||
757 | void (*set_pmu_cmdline_args_cpu_freq)(struct nvgpu_pmu *pmu, | ||
758 | u32 freq); | ||
759 | void (*set_pmu_cmdline_args_trace_size)(struct nvgpu_pmu *pmu, | ||
760 | u32 size); | ||
761 | void (*set_pmu_cmdline_args_trace_dma_base)( | ||
762 | struct nvgpu_pmu *pmu); | ||
763 | void (*config_pmu_cmdline_args_super_surface)( | ||
764 | struct nvgpu_pmu *pmu); | ||
765 | void (*set_pmu_cmdline_args_trace_dma_idx)( | ||
766 | struct nvgpu_pmu *pmu, u32 idx); | ||
767 | void * (*get_pmu_cmdline_args_ptr)(struct nvgpu_pmu *pmu); | ||
768 | u32 (*get_pmu_allocation_struct_size)(struct nvgpu_pmu *pmu); | ||
769 | void (*set_pmu_allocation_ptr)(struct nvgpu_pmu *pmu, | ||
770 | void **pmu_alloc_ptr, void *assign_ptr); | ||
771 | void (*pmu_allocation_set_dmem_size)(struct nvgpu_pmu *pmu, | ||
772 | void *pmu_alloc_ptr, u16 size); | ||
773 | u16 (*pmu_allocation_get_dmem_size)(struct nvgpu_pmu *pmu, | ||
774 | void *pmu_alloc_ptr); | ||
775 | u32 (*pmu_allocation_get_dmem_offset)(struct nvgpu_pmu *pmu, | ||
776 | void *pmu_alloc_ptr); | ||
777 | u32 * (*pmu_allocation_get_dmem_offset_addr)( | ||
778 | struct nvgpu_pmu *pmu, void *pmu_alloc_ptr); | ||
779 | void (*pmu_allocation_set_dmem_offset)(struct nvgpu_pmu *pmu, | ||
780 | void *pmu_alloc_ptr, u32 offset); | ||
781 | void * (*pmu_allocation_get_fb_addr)( | ||
782 | struct nvgpu_pmu *pmu, void *pmu_alloc_ptr); | ||
783 | u32 (*pmu_allocation_get_fb_size)( | ||
784 | struct nvgpu_pmu *pmu, void *pmu_alloc_ptr); | ||
785 | void (*get_pmu_init_msg_pmu_queue_params)( | ||
786 | struct nvgpu_falcon_queue *queue, u32 id, | ||
787 | void *pmu_init_msg); | ||
788 | void *(*get_pmu_msg_pmu_init_msg_ptr)( | ||
789 | struct pmu_init_msg *init); | ||
790 | u16 (*get_pmu_init_msg_pmu_sw_mg_off)( | ||
791 | union pmu_init_msg_pmu *init_msg); | ||
792 | u16 (*get_pmu_init_msg_pmu_sw_mg_size)( | ||
793 | union pmu_init_msg_pmu *init_msg); | ||
794 | u32 (*get_pmu_perfmon_cmd_start_size)(void); | ||
795 | int (*get_perfmon_cmd_start_offsetofvar)( | ||
796 | enum pmu_perfmon_cmd_start_fields field); | ||
797 | void (*perfmon_start_set_cmd_type)(struct pmu_perfmon_cmd *pc, | ||
798 | u8 value); | ||
799 | void (*perfmon_start_set_group_id)(struct pmu_perfmon_cmd *pc, | ||
800 | u8 value); | ||
801 | void (*perfmon_start_set_state_id)(struct pmu_perfmon_cmd *pc, | ||
802 | u8 value); | ||
803 | void (*perfmon_start_set_flags)(struct pmu_perfmon_cmd *pc, | ||
804 | u8 value); | ||
805 | u8 (*perfmon_start_get_flags)(struct pmu_perfmon_cmd *pc); | ||
806 | u32 (*get_pmu_perfmon_cmd_init_size)(void); | ||
807 | int (*get_perfmon_cmd_init_offsetofvar)( | ||
808 | enum pmu_perfmon_cmd_start_fields field); | ||
809 | void (*perfmon_cmd_init_set_sample_buffer)( | ||
810 | struct pmu_perfmon_cmd *pc, u16 value); | ||
811 | void (*perfmon_cmd_init_set_dec_cnt)( | ||
812 | struct pmu_perfmon_cmd *pc, u8 value); | ||
813 | void (*perfmon_cmd_init_set_base_cnt_id)( | ||
814 | struct pmu_perfmon_cmd *pc, u8 value); | ||
815 | void (*perfmon_cmd_init_set_samp_period_us)( | ||
816 | struct pmu_perfmon_cmd *pc, u32 value); | ||
817 | void (*perfmon_cmd_init_set_num_cnt)(struct pmu_perfmon_cmd *pc, | ||
818 | u8 value); | ||
819 | void (*perfmon_cmd_init_set_mov_avg)(struct pmu_perfmon_cmd *pc, | ||
820 | u8 value); | ||
821 | void *(*get_pmu_seq_in_a_ptr)( | ||
822 | struct pmu_sequence *seq); | ||
823 | void *(*get_pmu_seq_out_a_ptr)( | ||
824 | struct pmu_sequence *seq); | ||
825 | void (*set_pmu_cmdline_args_secure_mode)(struct nvgpu_pmu *pmu, | ||
826 | u32 val); | ||
827 | u32 (*get_perfmon_cntr_sz)(struct nvgpu_pmu *pmu); | ||
828 | void * (*get_perfmon_cntr_ptr)(struct nvgpu_pmu *pmu); | ||
829 | void (*set_perfmon_cntr_ut)(struct nvgpu_pmu *pmu, u16 ut); | ||
830 | void (*set_perfmon_cntr_lt)(struct nvgpu_pmu *pmu, u16 lt); | ||
831 | void (*set_perfmon_cntr_valid)(struct nvgpu_pmu *pmu, u8 val); | ||
832 | void (*set_perfmon_cntr_index)(struct nvgpu_pmu *pmu, u8 val); | ||
833 | void (*set_perfmon_cntr_group_id)(struct nvgpu_pmu *pmu, | ||
834 | u8 gid); | ||
835 | |||
836 | u8 (*pg_cmd_eng_buf_load_size)(struct pmu_pg_cmd *pg); | ||
837 | void (*pg_cmd_eng_buf_load_set_cmd_type)(struct pmu_pg_cmd *pg, | ||
838 | u8 value); | ||
839 | void (*pg_cmd_eng_buf_load_set_engine_id)(struct pmu_pg_cmd *pg, | ||
840 | u8 value); | ||
841 | void (*pg_cmd_eng_buf_load_set_buf_idx)(struct pmu_pg_cmd *pg, | ||
842 | u8 value); | ||
843 | void (*pg_cmd_eng_buf_load_set_pad)(struct pmu_pg_cmd *pg, | ||
844 | u8 value); | ||
845 | void (*pg_cmd_eng_buf_load_set_buf_size)(struct pmu_pg_cmd *pg, | ||
846 | u16 value); | ||
847 | void (*pg_cmd_eng_buf_load_set_dma_base)(struct pmu_pg_cmd *pg, | ||
848 | u32 value); | ||
849 | void (*pg_cmd_eng_buf_load_set_dma_offset)(struct pmu_pg_cmd *pg, | ||
850 | u8 value); | ||
851 | void (*pg_cmd_eng_buf_load_set_dma_idx)(struct pmu_pg_cmd *pg, | ||
852 | u8 value); | ||
853 | struct { | ||
854 | int (*boardobjgrp_pmucmd_construct_impl) | ||
855 | (struct gk20a *g, | ||
856 | struct boardobjgrp *pboardobjgrp, | ||
857 | struct boardobjgrp_pmu_cmd *cmd, u8 id, u8 msgid, | ||
858 | u8 hdrsize, u8 entrysize, u16 fbsize, u32 ss_offset, | ||
859 | u8 rpc_func_id); | ||
860 | int (*boardobjgrp_pmuset_impl)(struct gk20a *g, | ||
861 | struct boardobjgrp *pboardobjgrp); | ||
862 | int (*boardobjgrp_pmugetstatus_impl)(struct gk20a *g, | ||
863 | struct boardobjgrp *pboardobjgrp, | ||
864 | struct boardobjgrpmask *mask); | ||
865 | int (*is_boardobjgrp_pmucmd_id_valid)(struct gk20a *g, | ||
866 | struct boardobjgrp *pboardobjgrp, | ||
867 | struct boardobjgrp_pmu_cmd *cmd); | ||
868 | } boardobj; | ||
869 | struct { | ||
870 | u32 (*volt_set_voltage)(struct gk20a *g, | ||
871 | u32 logic_voltage_uv, u32 sram_voltage_uv); | ||
872 | u32 (*volt_get_voltage)(struct gk20a *g, | ||
873 | u8 volt_domain, u32 *pvoltage_uv); | ||
874 | u32 (*volt_send_load_cmd_to_pmu)(struct gk20a *g); | ||
875 | } volt; | ||
876 | struct { | ||
877 | u32 (*get_vbios_clk_domain)(u32 vbios_domain); | ||
878 | u32 (*clk_avfs_get_vin_cal_data)(struct gk20a *g, | ||
879 | struct avfsvinobjs *pvinobjs, | ||
880 | struct vin_device_v20 *pvindev); | ||
881 | u32 (*clk_vf_change_inject_data_fill)(struct gk20a *g, | ||
882 | struct nv_pmu_clk_rpc *rpccall, | ||
883 | struct set_fll_clk *setfllclk); | ||
884 | u32 (*perf_pmu_vfe_load)(struct gk20a *g); | ||
885 | u32 (*clk_set_boot_clk)(struct gk20a *g); | ||
886 | }clk; | ||
887 | } pmu_ver; | ||
888 | struct { | ||
889 | int (*get_netlist_name)(struct gk20a *g, int index, char *name); | ||
890 | bool (*is_fw_defined)(void); | ||
891 | } gr_ctx; | ||
892 | #ifdef CONFIG_GK20A_CTXSW_TRACE | ||
893 | /* | ||
894 | * Currently only supported on Linux due to the extremely tight | ||
895 | * integration with Linux device driver structure (in particular | ||
896 | * mmap). | ||
897 | */ | ||
898 | struct { | ||
899 | int (*init)(struct gk20a *g); | ||
900 | int (*max_entries)(struct gk20a *, | ||
901 | struct nvgpu_gpu_ctxsw_trace_filter *filter); | ||
902 | int (*flush)(struct gk20a *g); | ||
903 | int (*poll)(struct gk20a *g); | ||
904 | int (*enable)(struct gk20a *g); | ||
905 | int (*disable)(struct gk20a *g); | ||
906 | bool (*is_enabled)(struct gk20a *g); | ||
907 | int (*reset)(struct gk20a *g); | ||
908 | int (*bind_channel)(struct gk20a *g, struct channel_gk20a *ch); | ||
909 | int (*unbind_channel)(struct gk20a *g, | ||
910 | struct channel_gk20a *ch); | ||
911 | int (*deinit)(struct gk20a *g); | ||
912 | int (*alloc_user_buffer)(struct gk20a *g, | ||
913 | void **buf, size_t *size); | ||
914 | int (*free_user_buffer)(struct gk20a *g); | ||
915 | int (*mmap_user_buffer)(struct gk20a *g, | ||
916 | struct vm_area_struct *vma); | ||
917 | int (*set_filter)(struct gk20a *g, | ||
918 | struct nvgpu_gpu_ctxsw_trace_filter *filter); | ||
919 | } fecs_trace; | ||
920 | #endif | ||
921 | struct { | ||
922 | bool (*support_sparse)(struct gk20a *g); | ||
923 | u64 (*gmmu_map)(struct vm_gk20a *vm, | ||
924 | u64 map_offset, | ||
925 | struct nvgpu_sgt *sgt, | ||
926 | u64 buffer_offset, | ||
927 | u64 size, | ||
928 | u32 pgsz_idx, | ||
929 | u8 kind_v, | ||
930 | u32 ctag_offset, | ||
931 | u32 flags, | ||
932 | enum gk20a_mem_rw_flag rw_flag, | ||
933 | bool clear_ctags, | ||
934 | bool sparse, | ||
935 | bool priv, | ||
936 | struct vm_gk20a_mapping_batch *batch, | ||
937 | enum nvgpu_aperture aperture); | ||
938 | void (*gmmu_unmap)(struct vm_gk20a *vm, | ||
939 | u64 vaddr, | ||
940 | u64 size, | ||
941 | u32 pgsz_idx, | ||
942 | bool va_allocated, | ||
943 | enum gk20a_mem_rw_flag rw_flag, | ||
944 | bool sparse, | ||
945 | struct vm_gk20a_mapping_batch *batch); | ||
946 | int (*vm_bind_channel)(struct vm_gk20a *vm, | ||
947 | struct channel_gk20a *ch); | ||
948 | int (*fb_flush)(struct gk20a *g); | ||
949 | void (*l2_invalidate)(struct gk20a *g); | ||
950 | void (*l2_flush)(struct gk20a *g, bool invalidate); | ||
951 | void (*cbc_clean)(struct gk20a *g); | ||
952 | void (*set_big_page_size)(struct gk20a *g, | ||
953 | struct nvgpu_mem *mem, int size); | ||
954 | u32 (*get_big_page_sizes)(void); | ||
955 | u32 (*get_default_big_page_size)(void); | ||
956 | u32 (*get_iommu_bit)(struct gk20a *g); | ||
957 | int (*init_mm_setup_hw)(struct gk20a *g); | ||
958 | bool (*is_bar1_supported)(struct gk20a *g); | ||
959 | int (*init_bar2_vm)(struct gk20a *g); | ||
960 | void (*remove_bar2_vm)(struct gk20a *g); | ||
961 | const struct gk20a_mmu_level * | ||
962 | (*get_mmu_levels)(struct gk20a *g, u32 big_page_size); | ||
963 | void (*init_pdb)(struct gk20a *g, struct nvgpu_mem *inst_block, | ||
964 | struct vm_gk20a *vm); | ||
965 | u64 (*gpu_phys_addr)(struct gk20a *g, | ||
966 | struct nvgpu_gmmu_attrs *attrs, u64 phys); | ||
967 | int (*alloc_inst_block)(struct gk20a *g, | ||
968 | struct nvgpu_mem *inst_block); | ||
969 | void (*init_inst_block)(struct nvgpu_mem *inst_block, | ||
970 | struct vm_gk20a *vm, u32 big_page_size); | ||
971 | bool (*mmu_fault_pending)(struct gk20a *g); | ||
972 | void (*fault_info_mem_destroy)(struct gk20a *g); | ||
973 | void (*mmu_fault_disable_hw)(struct gk20a *g); | ||
974 | u32 (*get_kind_invalid)(void); | ||
975 | u32 (*get_kind_pitch)(void); | ||
976 | u32 (*get_flush_retries)(struct gk20a *g, | ||
977 | enum nvgpu_flush_op op); | ||
978 | } mm; | ||
979 | /* | ||
980 | * This function is called to allocate secure memory (memory | ||
981 | * that the CPU cannot see). The function should fill the | ||
982 | * context buffer descriptor (especially fields destroy, sgt, | ||
983 | * size). | ||
984 | */ | ||
985 | int (*secure_alloc)(struct gk20a *g, | ||
986 | struct gr_ctx_buffer_desc *desc, | ||
987 | size_t size); | ||
988 | struct { | ||
989 | void (*exit)(struct gk20a *g, struct nvgpu_mem *mem, | ||
990 | struct nvgpu_sgl *sgl); | ||
991 | u32 (*data032_r)(u32 i); | ||
992 | } pramin; | ||
993 | struct { | ||
994 | int (*init_therm_setup_hw)(struct gk20a *g); | ||
995 | void (*init_elcg_mode)(struct gk20a *g, u32 mode, u32 engine); | ||
996 | void (*init_blcg_mode)(struct gk20a *g, u32 mode, u32 engine); | ||
997 | int (*elcg_init_idle_filters)(struct gk20a *g); | ||
998 | #ifdef CONFIG_DEBUG_FS | ||
999 | void (*therm_debugfs_init)(struct gk20a *g); | ||
1000 | #endif | ||
1001 | int (*get_internal_sensor_curr_temp)(struct gk20a *g, u32 *temp_f24_8); | ||
1002 | void (*get_internal_sensor_limits)(s32 *max_24_8, | ||
1003 | s32 *min_24_8); | ||
1004 | u32 (*configure_therm_alert)(struct gk20a *g, s32 curr_warn_temp); | ||
1005 | } therm; | ||
1006 | struct { | ||
1007 | bool (*is_pmu_supported)(struct gk20a *g); | ||
1008 | int (*prepare_ucode)(struct gk20a *g); | ||
1009 | int (*pmu_setup_hw_and_bootstrap)(struct gk20a *g); | ||
1010 | int (*pmu_nsbootstrap)(struct nvgpu_pmu *pmu); | ||
1011 | int (*pmu_init_perfmon)(struct nvgpu_pmu *pmu); | ||
1012 | int (*pmu_perfmon_start_sampling)(struct nvgpu_pmu *pmu); | ||
1013 | int (*pmu_perfmon_stop_sampling)(struct nvgpu_pmu *pmu); | ||
1014 | int (*pmu_perfmon_get_samples_rpc)(struct nvgpu_pmu *pmu); | ||
1015 | int (*pmu_setup_elpg)(struct gk20a *g); | ||
1016 | u32 (*pmu_get_queue_head)(u32 i); | ||
1017 | u32 (*pmu_get_queue_head_size)(void); | ||
1018 | u32 (*pmu_get_queue_tail_size)(void); | ||
1019 | u32 (*pmu_get_queue_tail)(u32 i); | ||
1020 | int (*pmu_queue_head)(struct gk20a *g, | ||
1021 | struct nvgpu_falcon_queue *queue, u32 *head, bool set); | ||
1022 | int (*pmu_queue_tail)(struct gk20a *g, | ||
1023 | struct nvgpu_falcon_queue *queue, u32 *tail, bool set); | ||
1024 | void (*pmu_msgq_tail)(struct nvgpu_pmu *pmu, | ||
1025 | u32 *tail, bool set); | ||
1026 | u32 (*pmu_mutex_size)(void); | ||
1027 | int (*pmu_mutex_acquire)(struct nvgpu_pmu *pmu, | ||
1028 | u32 id, u32 *token); | ||
1029 | int (*pmu_mutex_release)(struct nvgpu_pmu *pmu, | ||
1030 | u32 id, u32 *token); | ||
1031 | bool (*pmu_is_interrupted)(struct nvgpu_pmu *pmu); | ||
1032 | void (*pmu_isr)(struct gk20a *g); | ||
1033 | void (*pmu_init_perfmon_counter)(struct gk20a *g); | ||
1034 | void (*pmu_pg_idle_counter_config)(struct gk20a *g, u32 pg_engine_id); | ||
1035 | u32 (*pmu_read_idle_counter)(struct gk20a *g, u32 counter_id); | ||
1036 | void (*pmu_reset_idle_counter)(struct gk20a *g, u32 counter_id); | ||
1037 | void (*pmu_dump_elpg_stats)(struct nvgpu_pmu *pmu); | ||
1038 | void (*pmu_dump_falcon_stats)(struct nvgpu_pmu *pmu); | ||
1039 | void (*pmu_enable_irq)(struct nvgpu_pmu *pmu, bool enable); | ||
1040 | int (*init_wpr_region)(struct gk20a *g); | ||
1041 | int (*load_lsfalcon_ucode)(struct gk20a *g, u32 falconidmask); | ||
1042 | void (*write_dmatrfbase)(struct gk20a *g, u32 addr); | ||
1043 | void (*pmu_elpg_statistics)(struct gk20a *g, u32 pg_engine_id, | ||
1044 | struct pmu_pg_stats_data *pg_stat_data); | ||
1045 | int (*pmu_pg_init_param)(struct gk20a *g, u32 pg_engine_id); | ||
1046 | int (*pmu_pg_set_sub_feature_mask)(struct gk20a *g, | ||
1047 | u32 pg_engine_id); | ||
1048 | u32 (*pmu_pg_supported_engines_list)(struct gk20a *g); | ||
1049 | u32 (*pmu_pg_engines_feature_list)(struct gk20a *g, | ||
1050 | u32 pg_engine_id); | ||
1051 | bool (*pmu_is_lpwr_feature_supported)(struct gk20a *g, | ||
1052 | u32 feature_id); | ||
1053 | int (*pmu_lpwr_enable_pg)(struct gk20a *g, bool pstate_lock); | ||
1054 | int (*pmu_lpwr_disable_pg)(struct gk20a *g, bool pstate_lock); | ||
1055 | u32 (*pmu_pg_param_post_init)(struct gk20a *g); | ||
1056 | void (*dump_secure_fuses)(struct gk20a *g); | ||
1057 | int (*reset_engine)(struct gk20a *g, bool do_reset); | ||
1058 | bool (*is_engine_in_reset)(struct gk20a *g); | ||
1059 | int (*falcon_wait_for_halt)(struct gk20a *g, | ||
1060 | unsigned int timeout); | ||
1061 | int (*falcon_clear_halt_interrupt_status)(struct gk20a *g, | ||
1062 | unsigned int timeout); | ||
1063 | int (*init_falcon_setup_hw)(struct gk20a *g, | ||
1064 | void *desc, u32 bl_sz); | ||
1065 | bool (*is_lazy_bootstrap)(u32 falcon_id); | ||
1066 | bool (*is_priv_load)(u32 falcon_id); | ||
1067 | void (*get_wpr)(struct gk20a *g, struct wpr_carveout_info *inf); | ||
1068 | int (*alloc_blob_space)(struct gk20a *g, | ||
1069 | size_t size, struct nvgpu_mem *mem); | ||
1070 | int (*pmu_populate_loader_cfg)(struct gk20a *g, | ||
1071 | void *lsfm, u32 *p_bl_gen_desc_size); | ||
1072 | int (*flcn_populate_bl_dmem_desc)(struct gk20a *g, | ||
1073 | void *lsfm, u32 *p_bl_gen_desc_size, u32 falconid); | ||
1074 | void (*handle_ext_irq)(struct gk20a *g, u32 intr); | ||
1075 | void (*set_irqmask)(struct gk20a *g); | ||
1076 | void (*update_lspmu_cmdline_args)(struct gk20a *g); | ||
1077 | void (*setup_apertures)(struct gk20a *g); | ||
1078 | u32 (*get_irqdest)(struct gk20a *g); | ||
1079 | int (*alloc_super_surface)(struct gk20a *g, | ||
1080 | struct nvgpu_mem *super_surface, u32 size); | ||
1081 | bool (*is_debug_mode_enabled)(struct gk20a *g); | ||
1082 | } pmu; | ||
1083 | struct { | ||
1084 | int (*init_debugfs)(struct gk20a *g); | ||
1085 | void (*disable_slowboot)(struct gk20a *g); | ||
1086 | int (*init_clk_support)(struct gk20a *g); | ||
1087 | int (*suspend_clk_support)(struct gk20a *g); | ||
1088 | u32 (*get_crystal_clk_hz)(struct gk20a *g); | ||
1089 | int (*clk_domain_get_f_points)(struct gk20a *g, | ||
1090 | u32 clkapidomain, u32 *pfpointscount, | ||
1091 | u16 *pfreqpointsinmhz); | ||
1092 | unsigned long (*measure_freq)(struct gk20a *g, u32 api_domain); | ||
1093 | u32 (*get_rate_cntr)(struct gk20a *g, struct namemap_cfg *c); | ||
1094 | unsigned long (*get_rate)(struct gk20a *g, u32 api_domain); | ||
1095 | int (*set_rate)(struct gk20a *g, u32 api_domain, unsigned long rate); | ||
1096 | unsigned long (*get_fmax_at_vmin_safe)(struct gk20a *g); | ||
1097 | u32 (*get_ref_clock_rate)(struct gk20a *g); | ||
1098 | int (*predict_mv_at_hz_cur_tfloor)(struct clk_gk20a *clk, | ||
1099 | unsigned long rate); | ||
1100 | unsigned long (*get_maxrate)(struct gk20a *g, u32 api_domain); | ||
1101 | int (*prepare_enable)(struct clk_gk20a *clk); | ||
1102 | void (*disable_unprepare)(struct clk_gk20a *clk); | ||
1103 | int (*get_voltage)(struct clk_gk20a *clk, u64 *val); | ||
1104 | int (*get_gpcclk_clock_counter)(struct clk_gk20a *clk, u64 *val); | ||
1105 | int (*pll_reg_write)(struct gk20a *g, u32 reg, u32 val); | ||
1106 | int (*get_pll_debug_data)(struct gk20a *g, | ||
1107 | struct nvgpu_clk_pll_debug_data *d); | ||
1108 | int (*mclk_init)(struct gk20a *g); | ||
1109 | void (*mclk_deinit)(struct gk20a *g); | ||
1110 | int (*mclk_change)(struct gk20a *g, u16 val); | ||
1111 | bool split_rail_support; | ||
1112 | bool support_clk_freq_controller; | ||
1113 | bool support_pmgr_domain; | ||
1114 | bool support_lpwr_pg; | ||
1115 | } clk; | ||
1116 | struct { | ||
1117 | int (*arbiter_clk_init)(struct gk20a *g); | ||
1118 | u32 (*get_arbiter_clk_domains)(struct gk20a *g); | ||
1119 | int (*get_arbiter_clk_range)(struct gk20a *g, u32 api_domain, | ||
1120 | u16 *min_mhz, u16 *max_mhz); | ||
1121 | int (*get_arbiter_clk_default)(struct gk20a *g, u32 api_domain, | ||
1122 | u16 *default_mhz); | ||
1123 | void (*clk_arb_run_arbiter_cb)(struct nvgpu_clk_arb *arb); | ||
1124 | /* This function is inherently unsafe to call while | ||
1125 | * arbiter is running arbiter must be blocked | ||
1126 | * before calling this function */ | ||
1127 | int (*get_current_pstate)(struct gk20a *g); | ||
1128 | void (*clk_arb_cleanup)(struct nvgpu_clk_arb *arb); | ||
1129 | } clk_arb; | ||
1130 | struct { | ||
1131 | int (*handle_pmu_perf_event)(struct gk20a *g, void *pmu_msg); | ||
1132 | } perf; | ||
1133 | struct { | ||
1134 | int (*exec_regops)(struct dbg_session_gk20a *dbg_s, | ||
1135 | struct nvgpu_dbg_reg_op *ops, | ||
1136 | u64 num_ops, | ||
1137 | bool *is_current_ctx); | ||
1138 | const struct regop_offset_range* ( | ||
1139 | *get_global_whitelist_ranges)(void); | ||
1140 | u64 (*get_global_whitelist_ranges_count)(void); | ||
1141 | const struct regop_offset_range* ( | ||
1142 | *get_context_whitelist_ranges)(void); | ||
1143 | u64 (*get_context_whitelist_ranges_count)(void); | ||
1144 | const u32* (*get_runcontrol_whitelist)(void); | ||
1145 | u64 (*get_runcontrol_whitelist_count)(void); | ||
1146 | const u32* (*get_qctl_whitelist)(void); | ||
1147 | u64 (*get_qctl_whitelist_count)(void); | ||
1148 | } regops; | ||
1149 | struct { | ||
1150 | void (*intr_mask)(struct gk20a *g); | ||
1151 | void (*intr_enable)(struct gk20a *g); | ||
1152 | void (*intr_unit_config)(struct gk20a *g, | ||
1153 | bool enable, bool is_stalling, u32 unit); | ||
1154 | void (*isr_stall)(struct gk20a *g); | ||
1155 | bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); | ||
1156 | bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr); | ||
1157 | bool (*is_stall_and_eng_intr_pending)(struct gk20a *g, | ||
1158 | u32 act_eng_id, u32 *eng_intr_pending); | ||
1159 | u32 (*intr_stall)(struct gk20a *g); | ||
1160 | void (*intr_stall_pause)(struct gk20a *g); | ||
1161 | void (*intr_stall_resume)(struct gk20a *g); | ||
1162 | u32 (*intr_nonstall)(struct gk20a *g); | ||
1163 | void (*intr_nonstall_pause)(struct gk20a *g); | ||
1164 | void (*intr_nonstall_resume)(struct gk20a *g); | ||
1165 | u32 (*isr_nonstall)(struct gk20a *g); | ||
1166 | void (*enable)(struct gk20a *g, u32 units); | ||
1167 | void (*disable)(struct gk20a *g, u32 units); | ||
1168 | void (*reset)(struct gk20a *g, u32 units); | ||
1169 | u32 (*boot_0)(struct gk20a *g, u32 *arch, u32 *impl, u32 *rev); | ||
1170 | bool (*is_intr1_pending)(struct gk20a *g, enum nvgpu_unit unit, u32 mc_intr_1); | ||
1171 | void (*log_pending_intrs)(struct gk20a *g); | ||
1172 | } mc; | ||
1173 | struct { | ||
1174 | void (*show_dump)(struct gk20a *g, | ||
1175 | struct gk20a_debug_output *o); | ||
1176 | } debug; | ||
1177 | struct { | ||
1178 | int (*dbg_set_powergate)(struct dbg_session_gk20a *dbg_s, | ||
1179 | bool disable_powergate); | ||
1180 | bool (*check_and_set_global_reservation)( | ||
1181 | struct dbg_session_gk20a *dbg_s, | ||
1182 | struct dbg_profiler_object_data *prof_obj); | ||
1183 | bool (*check_and_set_context_reservation)( | ||
1184 | struct dbg_session_gk20a *dbg_s, | ||
1185 | struct dbg_profiler_object_data *prof_obj); | ||
1186 | void (*release_profiler_reservation)( | ||
1187 | struct dbg_session_gk20a *dbg_s, | ||
1188 | struct dbg_profiler_object_data *prof_obj); | ||
1189 | int (*perfbuffer_enable)(struct gk20a *g, u64 offset, u32 size); | ||
1190 | int (*perfbuffer_disable)(struct gk20a *g); | ||
1191 | } dbg_session_ops; | ||
1192 | |||
1193 | int (*get_litter_value)(struct gk20a *g, int value); | ||
1194 | int (*chip_init_gpu_characteristics)(struct gk20a *g); | ||
1195 | |||
1196 | struct { | ||
1197 | void (*init_hw)(struct gk20a *g); | ||
1198 | void (*isr)(struct gk20a *g); | ||
1199 | int (*bar1_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst); | ||
1200 | int (*bar2_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst); | ||
1201 | u32 (*set_bar0_window)(struct gk20a *g, struct nvgpu_mem *mem, | ||
1202 | struct nvgpu_sgt *sgt, struct nvgpu_sgl *sgl, | ||
1203 | u32 w); | ||
1204 | u32 (*read_sw_scratch)(struct gk20a *g, u32 index); | ||
1205 | void (*write_sw_scratch)(struct gk20a *g, u32 index, u32 val); | ||
1206 | } bus; | ||
1207 | |||
1208 | struct { | ||
1209 | void (*isr)(struct gk20a *g); | ||
1210 | int (*read_ptimer)(struct gk20a *g, u64 *value); | ||
1211 | int (*get_timestamps_zipper)(struct gk20a *g, | ||
1212 | u32 source_id, u32 count, | ||
1213 | struct nvgpu_cpu_time_correlation_sample *); | ||
1214 | } ptimer; | ||
1215 | |||
1216 | struct { | ||
1217 | int (*init)(struct gk20a *g); | ||
1218 | int (*preos_wait_for_halt)(struct gk20a *g); | ||
1219 | void (*preos_reload_check)(struct gk20a *g); | ||
1220 | int (*devinit)(struct gk20a *g); | ||
1221 | int (*preos)(struct gk20a *g); | ||
1222 | int (*verify_devinit)(struct gk20a *g); | ||
1223 | } bios; | ||
1224 | |||
1225 | #if defined(CONFIG_GK20A_CYCLE_STATS) | ||
1226 | struct { | ||
1227 | int (*enable_snapshot)(struct channel_gk20a *ch, | ||
1228 | struct gk20a_cs_snapshot_client *client); | ||
1229 | void (*disable_snapshot)(struct gr_gk20a *gr); | ||
1230 | int (*check_data_available)(struct channel_gk20a *ch, | ||
1231 | u32 *pending, | ||
1232 | bool *hw_overflow); | ||
1233 | void (*set_handled_snapshots)(struct gk20a *g, u32 num); | ||
1234 | u32 (*allocate_perfmon_ids)(struct gk20a_cs_snapshot *data, | ||
1235 | u32 count); | ||
1236 | u32 (*release_perfmon_ids)(struct gk20a_cs_snapshot *data, | ||
1237 | u32 start, | ||
1238 | u32 count); | ||
1239 | int (*detach_snapshot)(struct channel_gk20a *ch, | ||
1240 | struct gk20a_cs_snapshot_client *client); | ||
1241 | bool (*get_overflow_status)(struct gk20a *g); | ||
1242 | u32 (*get_pending_snapshots)(struct gk20a *g); | ||
1243 | } css; | ||
1244 | #endif | ||
1245 | struct { | ||
1246 | int (*get_speed)(struct gk20a *g, u32 *xve_link_speed); | ||
1247 | int (*set_speed)(struct gk20a *g, u32 xve_link_speed); | ||
1248 | void (*available_speeds)(struct gk20a *g, u32 *speed_mask); | ||
1249 | u32 (*xve_readl)(struct gk20a *g, u32 reg); | ||
1250 | void (*xve_writel)(struct gk20a *g, u32 reg, u32 val); | ||
1251 | void (*disable_aspm)(struct gk20a *g); | ||
1252 | void (*reset_gpu)(struct gk20a *g); | ||
1253 | #if defined(CONFIG_PCI_MSI) | ||
1254 | void (*rearm_msi)(struct gk20a *g); | ||
1255 | #endif | ||
1256 | void (*enable_shadow_rom)(struct gk20a *g); | ||
1257 | void (*disable_shadow_rom)(struct gk20a *g); | ||
1258 | u32 (*get_link_control_status)(struct gk20a *g); | ||
1259 | } xve; | ||
1260 | struct { | ||
1261 | int (*falcon_hal_sw_init)(struct nvgpu_falcon *flcn); | ||
1262 | } falcon; | ||
1263 | struct { | ||
1264 | void (*enable_priv_ring)(struct gk20a *g); | ||
1265 | void (*isr)(struct gk20a *g); | ||
1266 | void (*decode_error_code)(struct gk20a *g, u32 error_code); | ||
1267 | void (*set_ppriv_timeout_settings)(struct gk20a *g); | ||
1268 | u32 (*enum_ltc)(struct gk20a *g); | ||
1269 | } priv_ring; | ||
1270 | struct { | ||
1271 | int (*check_priv_security)(struct gk20a *g); | ||
1272 | bool (*is_opt_ecc_enable)(struct gk20a *g); | ||
1273 | bool (*is_opt_feature_override_disable)(struct gk20a *g); | ||
1274 | u32 (*fuse_status_opt_fbio)(struct gk20a *g); | ||
1275 | u32 (*fuse_status_opt_fbp)(struct gk20a *g); | ||
1276 | u32 (*fuse_status_opt_rop_l2_fbp)(struct gk20a *g, u32 fbp); | ||
1277 | u32 (*fuse_status_opt_tpc_gpc)(struct gk20a *g, u32 gpc); | ||
1278 | void (*fuse_ctrl_opt_tpc_gpc)(struct gk20a *g, u32 gpc, u32 val); | ||
1279 | u32 (*fuse_opt_sec_debug_en)(struct gk20a *g); | ||
1280 | u32 (*fuse_opt_priv_sec_en)(struct gk20a *g); | ||
1281 | u32 (*read_vin_cal_fuse_rev)(struct gk20a *g); | ||
1282 | u32 (*read_vin_cal_slope_intercept_fuse)(struct gk20a *g, | ||
1283 | u32 vin_id, u32 *slope, | ||
1284 | u32 *intercept); | ||
1285 | u32 (*read_vin_cal_gain_offset_fuse)(struct gk20a *g, | ||
1286 | u32 vin_id, s8 *gain, | ||
1287 | s8 *offset); | ||
1288 | } fuse; | ||
1289 | struct { | ||
1290 | int (*init)(struct gk20a *g); | ||
1291 | int (*discover_ioctrl)(struct gk20a *g); | ||
1292 | int (*discover_link)(struct gk20a *g); | ||
1293 | int (*isr)(struct gk20a *g); | ||
1294 | int (*rxdet)(struct gk20a *g, u32 link_id); | ||
1295 | int (*setup_pll)(struct gk20a *g, unsigned long link_mask); | ||
1296 | int (*minion_data_ready_en)(struct gk20a *g, | ||
1297 | unsigned long link_mask, bool sync); | ||
1298 | void (*get_connected_link_mask)(u32 *link_mask); | ||
1299 | void (*set_sw_war)(struct gk20a *g, u32 link_id); | ||
1300 | /* API */ | ||
1301 | int (*link_early_init)(struct gk20a *g, unsigned long mask); | ||
1302 | u32 (*link_get_mode)(struct gk20a *g, u32 link_id); | ||
1303 | u32 (*link_get_state)(struct gk20a *g, u32 link_id); | ||
1304 | int (*link_set_mode)(struct gk20a *g, u32 link_id, u32 mode); | ||
1305 | u32 (*get_sublink_mode)(struct gk20a *g, u32 link_id, | ||
1306 | bool is_rx_sublink); | ||
1307 | u32 (*get_rx_sublink_state)(struct gk20a *g, u32 link_id); | ||
1308 | u32 (*get_tx_sublink_state)(struct gk20a *g, u32 link_id); | ||
1309 | int (*set_sublink_mode)(struct gk20a *g, u32 link_id, | ||
1310 | bool is_rx_sublink, u32 mode); | ||
1311 | int (*interface_init)(struct gk20a *g); | ||
1312 | int (*interface_disable)(struct gk20a *g); | ||
1313 | int (*reg_init)(struct gk20a *g); | ||
1314 | int (*shutdown)(struct gk20a *g); | ||
1315 | int (*early_init)(struct gk20a *g); | ||
1316 | } nvlink; | ||
1317 | struct { | ||
1318 | u32 (*get_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g); | ||
1319 | void (*set_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g, u32 val); | ||
1320 | u32 (*get_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g); | ||
1321 | void (*set_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g, u32 val); | ||
1322 | } top; | ||
1323 | void (*semaphore_wakeup)(struct gk20a *g, bool post_events); | ||
1324 | }; | ||
1325 | |||
1326 | struct nvgpu_bios_ucode { | ||
1327 | u8 *bootloader; | ||
1328 | u32 bootloader_phys_base; | ||
1329 | u32 bootloader_size; | ||
1330 | u8 *ucode; | ||
1331 | u32 phys_base; | ||
1332 | u32 size; | ||
1333 | u8 *dmem; | ||
1334 | u32 dmem_phys_base; | ||
1335 | u32 dmem_size; | ||
1336 | u32 code_entry_point; | ||
1337 | }; | ||
1338 | |||
1339 | struct nvgpu_bios { | ||
1340 | u32 vbios_version; | ||
1341 | u8 vbios_oem_version; | ||
1342 | |||
1343 | u8 *data; | ||
1344 | size_t size; | ||
1345 | |||
1346 | struct nvgpu_bios_ucode devinit; | ||
1347 | struct nvgpu_bios_ucode preos; | ||
1348 | |||
1349 | u8 *devinit_tables; | ||
1350 | u32 devinit_tables_size; | ||
1351 | u8 *bootscripts; | ||
1352 | u32 bootscripts_size; | ||
1353 | |||
1354 | u8 mem_strap_data_count; | ||
1355 | u16 mem_strap_xlat_tbl_ptr; | ||
1356 | |||
1357 | u32 condition_table_ptr; | ||
1358 | |||
1359 | u32 devinit_tables_phys_base; | ||
1360 | u32 devinit_script_phys_base; | ||
1361 | |||
1362 | struct bit_token *perf_token; | ||
1363 | struct bit_token *clock_token; | ||
1364 | struct bit_token *virt_token; | ||
1365 | u32 expansion_rom_offset; | ||
1366 | |||
1367 | u32 nvlink_config_data_offset; | ||
1368 | }; | ||
1369 | |||
1370 | struct nvgpu_gpu_params { | ||
1371 | /* GPU architecture ID */ | ||
1372 | u32 gpu_arch; | ||
1373 | /* GPU implementation ID */ | ||
1374 | u32 gpu_impl; | ||
1375 | /* GPU revision ID */ | ||
1376 | u32 gpu_rev; | ||
1377 | /* sm version */ | ||
1378 | u32 sm_arch_sm_version; | ||
1379 | /* sm instruction set */ | ||
1380 | u32 sm_arch_spa_version; | ||
1381 | u32 sm_arch_warp_count; | ||
1382 | }; | ||
1383 | |||
1384 | struct gk20a { | ||
1385 | void (*free)(struct gk20a *g); | ||
1386 | struct nvgpu_nvhost_dev *nvhost_dev; | ||
1387 | |||
1388 | /* | ||
1389 | * Used by <nvgpu/enabled.h>. Do not access directly! | ||
1390 | */ | ||
1391 | unsigned long *enabled_flags; | ||
1392 | |||
1393 | nvgpu_atomic_t usage_count; | ||
1394 | |||
1395 | struct nvgpu_mutex ctxsw_disable_lock; | ||
1396 | int ctxsw_disable_count; | ||
1397 | |||
1398 | struct nvgpu_ref refcount; | ||
1399 | |||
1400 | const char *name; | ||
1401 | |||
1402 | bool gpu_reset_done; | ||
1403 | bool power_on; | ||
1404 | bool suspended; | ||
1405 | bool sw_ready; | ||
1406 | |||
1407 | u64 log_mask; | ||
1408 | u32 log_trace; | ||
1409 | |||
1410 | struct nvgpu_mutex tpc_pg_lock; | ||
1411 | |||
1412 | struct nvgpu_gpu_params params; | ||
1413 | |||
1414 | /* | ||
1415 | * Guards access to hardware when usual gk20a_{busy,idle} are skipped | ||
1416 | * for submits and held for channel lifetime but dropped for an ongoing | ||
1417 | * gk20a_do_idle(). | ||
1418 | */ | ||
1419 | struct nvgpu_rwsem deterministic_busy; | ||
1420 | |||
1421 | struct nvgpu_falcon pmu_flcn; | ||
1422 | struct nvgpu_falcon sec2_flcn; | ||
1423 | struct nvgpu_falcon fecs_flcn; | ||
1424 | struct nvgpu_falcon gpccs_flcn; | ||
1425 | struct nvgpu_falcon nvdec_flcn; | ||
1426 | struct nvgpu_falcon minion_flcn; | ||
1427 | struct clk_gk20a clk; | ||
1428 | struct fifo_gk20a fifo; | ||
1429 | struct nvgpu_nvlink_dev nvlink; | ||
1430 | struct gr_gk20a gr; | ||
1431 | struct sim_nvgpu *sim; | ||
1432 | struct mm_gk20a mm; | ||
1433 | struct nvgpu_pmu pmu; | ||
1434 | struct acr_desc acr; | ||
1435 | struct nvgpu_ecc ecc; | ||
1436 | struct clk_pmupstate clk_pmu; | ||
1437 | struct perf_pmupstate perf_pmu; | ||
1438 | struct pmgr_pmupstate pmgr_pmu; | ||
1439 | struct therm_pmupstate therm_pmu; | ||
1440 | |||
1441 | #ifdef CONFIG_DEBUG_FS | ||
1442 | struct railgate_stats pstats; | ||
1443 | #endif | ||
1444 | u32 gr_idle_timeout_default; | ||
1445 | bool timeouts_disabled_by_user; | ||
1446 | unsigned int ch_wdt_timeout_ms; | ||
1447 | u32 fifo_eng_timeout_us; | ||
1448 | |||
1449 | struct nvgpu_mutex poweron_lock; | ||
1450 | struct nvgpu_mutex poweroff_lock; | ||
1451 | |||
1452 | /* Channel priorities */ | ||
1453 | u32 timeslice_low_priority_us; | ||
1454 | u32 timeslice_medium_priority_us; | ||
1455 | u32 timeslice_high_priority_us; | ||
1456 | u32 min_timeslice_us; | ||
1457 | u32 max_timeslice_us; | ||
1458 | bool runlist_interleave; | ||
1459 | |||
1460 | bool slcg_enabled; | ||
1461 | bool blcg_enabled; | ||
1462 | bool elcg_enabled; | ||
1463 | bool elpg_enabled; | ||
1464 | bool aelpg_enabled; | ||
1465 | bool can_elpg; | ||
1466 | bool mscg_enabled; | ||
1467 | bool forced_idle; | ||
1468 | bool forced_reset; | ||
1469 | bool allow_all; | ||
1470 | |||
1471 | u32 ptimer_src_freq; | ||
1472 | |||
1473 | int railgate_delay; | ||
1474 | u8 ldiv_slowdown_factor; | ||
1475 | unsigned int aggressive_sync_destroy_thresh; | ||
1476 | bool aggressive_sync_destroy; | ||
1477 | |||
1478 | bool has_syncpoints; | ||
1479 | /* Debugfs knob for forcing syncpt support off in runtime. */ | ||
1480 | u32 disable_syncpoints; | ||
1481 | |||
1482 | bool support_pmu; | ||
1483 | u32 bootstrap_owner; | ||
1484 | |||
1485 | bool is_virtual; | ||
1486 | |||
1487 | bool has_cde; | ||
1488 | |||
1489 | u32 emc3d_ratio; | ||
1490 | |||
1491 | struct nvgpu_spinlock ltc_enabled_lock; | ||
1492 | |||
1493 | struct gk20a_ctxsw_ucode_info ctxsw_ucode_info; | ||
1494 | |||
1495 | /* | ||
1496 | * A group of semaphore pools. One for each channel. | ||
1497 | */ | ||
1498 | struct nvgpu_semaphore_sea *sema_sea; | ||
1499 | |||
1500 | /* held while manipulating # of debug/profiler sessions present */ | ||
1501 | /* also prevents debug sessions from attaching until released */ | ||
1502 | struct nvgpu_mutex dbg_sessions_lock; | ||
1503 | int dbg_powergating_disabled_refcount; /*refcount for pg disable */ | ||
1504 | /*refcount for timeout disable */ | ||
1505 | nvgpu_atomic_t timeouts_disabled_refcount; | ||
1506 | |||
1507 | /* must have dbg_sessions_lock before use */ | ||
1508 | struct nvgpu_dbg_reg_op *dbg_regops_tmp_buf; | ||
1509 | u32 dbg_regops_tmp_buf_ops; | ||
1510 | |||
1511 | /* For perfbuf mapping */ | ||
1512 | struct { | ||
1513 | struct dbg_session_gk20a *owner; | ||
1514 | u64 offset; | ||
1515 | } perfbuf; | ||
1516 | |||
1517 | /* For profiler reservations */ | ||
1518 | struct nvgpu_list_node profiler_objects; | ||
1519 | bool global_profiler_reservation_held; | ||
1520 | int profiler_reservation_count; | ||
1521 | |||
1522 | void (*remove_support)(struct gk20a *); | ||
1523 | |||
1524 | u64 pg_ingating_time_us; | ||
1525 | u64 pg_ungating_time_us; | ||
1526 | u32 pg_gating_cnt; | ||
1527 | |||
1528 | struct nvgpu_spinlock mc_enable_lock; | ||
1529 | |||
1530 | struct gk20a_as as; | ||
1531 | |||
1532 | struct nvgpu_mutex client_lock; | ||
1533 | int client_refcount; /* open channels and ctrl nodes */ | ||
1534 | |||
1535 | struct gpu_ops ops; | ||
1536 | u32 mc_intr_mask_restore[4]; | ||
1537 | /*used for change of enum zbc update cmd id from ver 0 to ver1*/ | ||
1538 | u32 pmu_ver_cmd_id_zbc_table_update; | ||
1539 | u32 pmu_lsf_pmu_wpr_init_done; | ||
1540 | u32 pmu_lsf_loaded_falcon_id; | ||
1541 | |||
1542 | int irqs_enabled; | ||
1543 | int irq_stall; /* can be same as irq_nonstall in case of PCI */ | ||
1544 | int irq_nonstall; | ||
1545 | u32 max_ltc_count; | ||
1546 | u32 ltc_count; | ||
1547 | u32 ltc_streamid; | ||
1548 | |||
1549 | struct gk20a_worker { | ||
1550 | struct nvgpu_thread poll_task; | ||
1551 | nvgpu_atomic_t put; | ||
1552 | struct nvgpu_cond wq; | ||
1553 | struct nvgpu_list_node items; | ||
1554 | struct nvgpu_spinlock items_lock; | ||
1555 | struct nvgpu_mutex start_lock; | ||
1556 | } channel_worker, clk_arb_worker; | ||
1557 | |||
1558 | struct { | ||
1559 | void (*open)(struct channel_gk20a *ch); | ||
1560 | void (*close)(struct channel_gk20a *ch); | ||
1561 | void (*work_completion_signal)(struct channel_gk20a *ch); | ||
1562 | void (*work_completion_cancel_sync)(struct channel_gk20a *ch); | ||
1563 | bool (*os_fence_framework_inst_exists)(struct channel_gk20a *ch); | ||
1564 | int (*init_os_fence_framework)( | ||
1565 | struct channel_gk20a *ch, const char *fmt, ...); | ||
1566 | void (*signal_os_fence_framework)(struct channel_gk20a *ch); | ||
1567 | void (*destroy_os_fence_framework)(struct channel_gk20a *ch); | ||
1568 | int (*copy_user_gpfifo)(struct nvgpu_gpfifo_entry *dest, | ||
1569 | struct nvgpu_gpfifo_userdata userdata, | ||
1570 | u32 start, u32 length); | ||
1571 | int (*alloc_usermode_buffers)(struct channel_gk20a *c, | ||
1572 | struct nvgpu_gpfifo_args *gpfifo_args); | ||
1573 | } os_channel; | ||
1574 | |||
1575 | struct gk20a_scale_profile *scale_profile; | ||
1576 | unsigned long last_freq; | ||
1577 | |||
1578 | struct gk20a_ctxsw_trace *ctxsw_trace; | ||
1579 | struct gk20a_fecs_trace *fecs_trace; | ||
1580 | |||
1581 | bool mmu_debug_ctrl; | ||
1582 | |||
1583 | u32 tpc_fs_mask_user; | ||
1584 | |||
1585 | u32 tpc_pg_mask; | ||
1586 | bool can_tpc_powergate; | ||
1587 | |||
1588 | u32 valid_tpc_mask; | ||
1589 | |||
1590 | struct nvgpu_bios bios; | ||
1591 | bool bios_is_init; | ||
1592 | |||
1593 | struct nvgpu_clk_arb *clk_arb; | ||
1594 | |||
1595 | struct nvgpu_mutex clk_arb_enable_lock; | ||
1596 | |||
1597 | struct gk20a_ce_app ce_app; | ||
1598 | |||
1599 | bool ltc_intr_en_illegal_compstat; | ||
1600 | |||
1601 | /* PCI device identifier */ | ||
1602 | u16 pci_vendor_id, pci_device_id; | ||
1603 | u16 pci_subsystem_vendor_id, pci_subsystem_device_id; | ||
1604 | u16 pci_class; | ||
1605 | u8 pci_revision; | ||
1606 | |||
1607 | /* | ||
1608 | * PCI power management: i2c device index, port and address for | ||
1609 | * INA3221. | ||
1610 | */ | ||
1611 | u32 ina3221_dcb_index; | ||
1612 | u32 ina3221_i2c_address; | ||
1613 | u32 ina3221_i2c_port; | ||
1614 | bool hardcode_sw_threshold; | ||
1615 | |||
1616 | /* PCIe power states. */ | ||
1617 | bool xve_l0s; | ||
1618 | bool xve_l1; | ||
1619 | |||
1620 | /* Current warning temp in sfxp24.8 */ | ||
1621 | s32 curr_warn_temp; | ||
1622 | |||
1623 | #if defined(CONFIG_PCI_MSI) | ||
1624 | /* Check if msi is enabled */ | ||
1625 | bool msi_enabled; | ||
1626 | #endif | ||
1627 | #ifdef CONFIG_NVGPU_TRACK_MEM_USAGE | ||
1628 | struct nvgpu_mem_alloc_tracker *vmallocs; | ||
1629 | struct nvgpu_mem_alloc_tracker *kmallocs; | ||
1630 | #endif | ||
1631 | |||
1632 | /* The minimum VBIOS version supported */ | ||
1633 | u32 vbios_min_version; | ||
1634 | |||
1635 | /* memory training sequence and mclk switch scripts */ | ||
1636 | u32 mem_config_idx; | ||
1637 | |||
1638 | u64 dma_memory_used; | ||
1639 | |||
1640 | #if defined(CONFIG_TEGRA_GK20A_NVHOST) | ||
1641 | u64 syncpt_unit_base; | ||
1642 | size_t syncpt_unit_size; | ||
1643 | u32 syncpt_size; | ||
1644 | #endif | ||
1645 | struct nvgpu_mem syncpt_mem; | ||
1646 | |||
1647 | struct nvgpu_list_node boardobj_head; | ||
1648 | struct nvgpu_list_node boardobjgrp_head; | ||
1649 | }; | ||
1650 | |||
1651 | static inline bool nvgpu_is_timeouts_enabled(struct gk20a *g) | ||
1652 | { | ||
1653 | return nvgpu_atomic_read(&g->timeouts_disabled_refcount) == 0; | ||
1654 | } | ||
1655 | |||
1656 | static inline u32 gk20a_get_gr_idle_timeout(struct gk20a *g) | ||
1657 | { | ||
1658 | return nvgpu_is_timeouts_enabled(g) ? | ||
1659 | g->gr_idle_timeout_default : UINT_MAX; | ||
1660 | } | ||
1661 | |||
1662 | #define MULTICHAR_TAG(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d)) | ||
1663 | enum BAR0_DEBUG_OPERATION { | ||
1664 | BARO_ZERO_NOP = 0, | ||
1665 | OP_END = MULTICHAR_TAG('D', 'O', 'N', 'E'), | ||
1666 | BAR0_READ32 = MULTICHAR_TAG('0', 'R', '3', '2'), | ||
1667 | BAR0_WRITE32 = MULTICHAR_TAG('0', 'W', '3', '2'), | ||
1668 | }; | ||
1669 | |||
1670 | struct share_buffer_head { | ||
1671 | enum BAR0_DEBUG_OPERATION operation; | ||
1672 | /* size of the operation item */ | ||
1673 | u32 size; | ||
1674 | u32 completed; | ||
1675 | u32 failed; | ||
1676 | u64 context; | ||
1677 | u64 completion_callback; | ||
1678 | }; | ||
1679 | |||
1680 | struct gk20a_cyclestate_buffer_elem { | ||
1681 | struct share_buffer_head head; | ||
1682 | /* in */ | ||
1683 | u64 p_data; | ||
1684 | u64 p_done; | ||
1685 | u32 offset_bar0; | ||
1686 | u16 first_bit; | ||
1687 | u16 last_bit; | ||
1688 | /* out */ | ||
1689 | /* keep 64 bits to be consistent */ | ||
1690 | u64 data; | ||
1691 | }; | ||
1692 | |||
1693 | /* operations that will need to be executed on non stall workqueue */ | ||
1694 | #define GK20A_NONSTALL_OPS_WAKEUP_SEMAPHORE BIT32(0) | ||
1695 | #define GK20A_NONSTALL_OPS_POST_EVENTS BIT32(1) | ||
1696 | |||
1697 | /* register accessors */ | ||
1698 | void __nvgpu_check_gpu_state(struct gk20a *g); | ||
1699 | void __gk20a_warn_on_no_regs(void); | ||
1700 | |||
1701 | /* classes that the device supports */ | ||
1702 | /* TBD: get these from an open-sourced SDK? */ | ||
1703 | enum { | ||
1704 | FERMI_TWOD_A = 0x902D, | ||
1705 | KEPLER_INLINE_TO_MEMORY_A = 0xA040, | ||
1706 | KEPLER_DMA_COPY_A = 0xA0B5, | ||
1707 | }; | ||
1708 | |||
1709 | #define GK20A_BAR0_IORESOURCE_MEM 0 | ||
1710 | #define GK20A_BAR1_IORESOURCE_MEM 1 | ||
1711 | #define GK20A_SIM_IORESOURCE_MEM 2 | ||
1712 | |||
1713 | void gk20a_busy_noresume(struct gk20a *g); | ||
1714 | void gk20a_idle_nosuspend(struct gk20a *g); | ||
1715 | int __must_check gk20a_busy(struct gk20a *g); | ||
1716 | void gk20a_idle(struct gk20a *g); | ||
1717 | int __gk20a_do_idle(struct gk20a *g, bool force_reset); | ||
1718 | int __gk20a_do_unidle(struct gk20a *g); | ||
1719 | |||
1720 | int gk20a_can_busy(struct gk20a *g); | ||
1721 | int gk20a_wait_for_idle(struct gk20a *g); | ||
1722 | |||
1723 | #define NVGPU_GPU_ARCHITECTURE_SHIFT 4 | ||
1724 | |||
1725 | /* constructs unique and compact GPUID from nvgpu_gpu_characteristics | ||
1726 | * arch/impl fields */ | ||
1727 | #define GK20A_GPUID(arch, impl) ((u32) ((arch) | (impl))) | ||
1728 | |||
1729 | #define GK20A_GPUID_GK20A 0x000000EA | ||
1730 | #define GK20A_GPUID_GM20B 0x0000012B | ||
1731 | #define GK20A_GPUID_GM20B_B 0x0000012E | ||
1732 | #define NVGPU_GPUID_GP10B 0x0000013B | ||
1733 | #define NVGPU_GPUID_GP104 0x00000134 | ||
1734 | #define NVGPU_GPUID_GP106 0x00000136 | ||
1735 | #define NVGPU_GPUID_GV11B 0x0000015B | ||
1736 | #define NVGPU_GPUID_GV100 0x00000140 | ||
1737 | |||
1738 | int gk20a_init_gpu_characteristics(struct gk20a *g); | ||
1739 | |||
1740 | int gk20a_prepare_poweroff(struct gk20a *g); | ||
1741 | int gk20a_finalize_poweron(struct gk20a *g); | ||
1742 | |||
1743 | void nvgpu_wait_for_deferred_interrupts(struct gk20a *g); | ||
1744 | |||
1745 | struct gk20a * __must_check gk20a_get(struct gk20a *g); | ||
1746 | void gk20a_put(struct gk20a *g); | ||
1747 | |||
1748 | static inline bool gk20a_platform_has_syncpoints(struct gk20a *g) | ||
1749 | { | ||
1750 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
1751 | return g->has_syncpoints && !g->disable_syncpoints; | ||
1752 | #else | ||
1753 | return false; | ||
1754 | #endif | ||
1755 | } | ||
1756 | |||
1757 | int gk20a_detect_chip(struct gk20a *g); | ||
1758 | #endif /* GK20A_H */ | ||