aboutsummaryrefslogtreecommitdiffstats
path: root/include/nvgpu/gk20a.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/nvgpu/gk20a.h')
-rw-r--r--include/nvgpu/gk20a.h1807
1 files changed, 0 insertions, 1807 deletions
diff --git a/include/nvgpu/gk20a.h b/include/nvgpu/gk20a.h
deleted file mode 100644
index 19bfaee..0000000
--- a/include/nvgpu/gk20a.h
+++ /dev/null
@@ -1,1807 +0,0 @@
1/*
2 * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved.
3 *
4 * GK20A Graphics
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24#ifndef GK20A_H
25#define GK20A_H
26
27struct gk20a;
28struct fifo_gk20a;
29struct channel_gk20a;
30struct gr_gk20a;
31struct sim_nvgpu;
32struct gk20a_ctxsw_ucode_segments;
33struct gk20a_fecs_trace;
34struct gk20a_ctxsw_trace;
35struct acr_desc;
36struct nvgpu_mem_alloc_tracker;
37struct dbg_profiler_object_data;
38struct gk20a_debug_output;
39struct nvgpu_clk_pll_debug_data;
40struct nvgpu_nvhost_dev;
41struct nvgpu_cpu_time_correlation_sample;
42struct nvgpu_mem_sgt;
43struct nvgpu_warpstate;
44struct nvgpu_clk_session;
45struct nvgpu_clk_arb;
46#ifdef CONFIG_GK20A_CTXSW_TRACE
47struct nvgpu_gpu_ctxsw_trace_filter;
48#endif
49struct priv_cmd_entry;
50struct nvgpu_setup_bind_args;
51
52#ifdef __KERNEL__
53#include <linux/notifier.h>
54#endif
55#include <nvgpu/lock.h>
56#include <nvgpu/thread.h>
57
58#include <nvgpu/mm.h>
59#include <nvgpu/as.h>
60#include <nvgpu/log.h>
61#include <nvgpu/pramin.h>
62#include <nvgpu/acr/nvgpu_acr.h>
63#include <nvgpu/kref.h>
64#include <nvgpu/falcon.h>
65#include <nvgpu/pmu.h>
66#include <nvgpu/atomic.h>
67#include <nvgpu/barrier.h>
68#include <nvgpu/rwsem.h>
69#include <nvgpu/nvlink.h>
70#include <nvgpu/sim.h>
71#include <nvgpu/ecc.h>
72#include <nvgpu/tsg.h>
73#include <nvgpu/sec2.h>
74#include <nvgpu/sched.h>
75
76#include "gk20a/clk_gk20a.h"
77#include "gk20a/ce2_gk20a.h"
78#include "gk20a/fifo_gk20a.h"
79#include "clk/clk.h"
80#include "pmu_perf/pmu_perf.h"
81#include "pmgr/pmgr.h"
82#include "therm/thrm.h"
83
84#ifdef CONFIG_DEBUG_FS
85struct railgate_stats {
86 unsigned long last_rail_gate_start;
87 unsigned long last_rail_gate_complete;
88 unsigned long last_rail_ungate_start;
89 unsigned long last_rail_ungate_complete;
90 unsigned long total_rail_gate_time_ms;
91 unsigned long total_rail_ungate_time_ms;
92 unsigned long railgating_cycle_count;
93};
94#endif
95
96enum gk20a_cbc_op {
97 gk20a_cbc_op_clear,
98 gk20a_cbc_op_clean,
99 gk20a_cbc_op_invalidate,
100};
101
102#define MC_INTR_UNIT_DISABLE false
103#define MC_INTR_UNIT_ENABLE true
104
105#define GPU_LIT_NUM_GPCS 0
106#define GPU_LIT_NUM_PES_PER_GPC 1
107#define GPU_LIT_NUM_ZCULL_BANKS 2
108#define GPU_LIT_NUM_TPC_PER_GPC 3
109#define GPU_LIT_NUM_SM_PER_TPC 4
110#define GPU_LIT_NUM_FBPS 5
111#define GPU_LIT_GPC_BASE 6
112#define GPU_LIT_GPC_STRIDE 7
113#define GPU_LIT_GPC_SHARED_BASE 8
114#define GPU_LIT_TPC_IN_GPC_BASE 9
115#define GPU_LIT_TPC_IN_GPC_STRIDE 10
116#define GPU_LIT_TPC_IN_GPC_SHARED_BASE 11
117#define GPU_LIT_PPC_IN_GPC_BASE 12
118#define GPU_LIT_PPC_IN_GPC_STRIDE 13
119#define GPU_LIT_PPC_IN_GPC_SHARED_BASE 14
120#define GPU_LIT_ROP_BASE 15
121#define GPU_LIT_ROP_STRIDE 16
122#define GPU_LIT_ROP_SHARED_BASE 17
123#define GPU_LIT_HOST_NUM_ENGINES 18
124#define GPU_LIT_HOST_NUM_PBDMA 19
125#define GPU_LIT_LTC_STRIDE 20
126#define GPU_LIT_LTS_STRIDE 21
127#define GPU_LIT_NUM_FBPAS 22
128#define GPU_LIT_FBPA_STRIDE 23
129#define GPU_LIT_FBPA_BASE 24
130#define GPU_LIT_FBPA_SHARED_BASE 25
131#define GPU_LIT_SM_PRI_STRIDE 26
132#define GPU_LIT_SMPC_PRI_BASE 27
133#define GPU_LIT_SMPC_PRI_SHARED_BASE 28
134#define GPU_LIT_SMPC_PRI_UNIQUE_BASE 29
135#define GPU_LIT_SMPC_PRI_STRIDE 30
136#define GPU_LIT_TWOD_CLASS 31
137#define GPU_LIT_THREED_CLASS 32
138#define GPU_LIT_COMPUTE_CLASS 33
139#define GPU_LIT_GPFIFO_CLASS 34
140#define GPU_LIT_I2M_CLASS 35
141#define GPU_LIT_DMA_COPY_CLASS 36
142#define GPU_LIT_GPC_PRIV_STRIDE 37
143#define GPU_LIT_PERFMON_PMMGPCTPCA_DOMAIN_START 38
144#define GPU_LIT_PERFMON_PMMGPCTPCB_DOMAIN_START 39
145#define GPU_LIT_PERFMON_PMMGPCTPC_DOMAIN_COUNT 40
146#define GPU_LIT_PERFMON_PMMFBP_LTC_DOMAIN_START 41
147#define GPU_LIT_PERFMON_PMMFBP_LTC_DOMAIN_COUNT 42
148#define GPU_LIT_PERFMON_PMMFBP_ROP_DOMAIN_START 43
149#define GPU_LIT_PERFMON_PMMFBP_ROP_DOMAIN_COUNT 44
150
151#define nvgpu_get_litter_value(g, v) (g)->ops.get_litter_value((g), v)
152
153#define MAX_TPC_PG_CONFIGS 9
154
155enum nvgpu_unit;
156
157enum nvgpu_flush_op;
158enum gk20a_mem_rw_flag;
159
160struct _resmgr_context;
161struct nvgpu_gpfifo_entry;
162
163struct nvgpu_gpfifo_userdata {
164 struct nvgpu_gpfifo_entry __user *entries;
165 struct _resmgr_context *context;
166};
167
168/*
169 * gpu_ops should only contain function pointers! Non-function pointer members
170 * should go in struct gk20a or be implemented with the boolean flag API defined
171 * in nvgpu/enabled.h
172 */
173
174/* index for FB fault buffer functions */
175#define NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX 0U
176#define NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX 1U
177#define NVGPU_FB_MMU_FAULT_BUF_DISABLED 0U
178#define NVGPU_FB_MMU_FAULT_BUF_ENABLED 1U
179
180/* Parameters for init_elcg_mode/init_blcg_mode */
181enum {
182 ELCG_RUN, /* clk always run, i.e. disable elcg */
183 ELCG_STOP, /* clk is stopped */
184 ELCG_AUTO /* clk will run when non-idle, standard elcg mode */
185};
186
187enum {
188 BLCG_RUN, /* clk always run, i.e. disable blcg */
189 BLCG_AUTO /* clk will run when non-idle, standard blcg mode */
190};
191
192struct gpu_ops {
193 struct {
194 int (*determine_L2_size_bytes)(struct gk20a *gk20a);
195 u64 (*get_cbc_base_divisor)(struct gk20a *g);
196 int (*init_comptags)(struct gk20a *g, struct gr_gk20a *gr);
197 int (*cbc_ctrl)(struct gk20a *g, enum gk20a_cbc_op op,
198 u32 min, u32 max);
199 void (*set_zbc_color_entry)(struct gk20a *g,
200 struct zbc_entry *color_val,
201 u32 index);
202 void (*set_zbc_depth_entry)(struct gk20a *g,
203 struct zbc_entry *depth_val,
204 u32 index);
205 void (*set_zbc_s_entry)(struct gk20a *g,
206 struct zbc_entry *s_val,
207 u32 index);
208 void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr);
209 void (*set_enabled)(struct gk20a *g, bool enabled);
210 void (*init_fs_state)(struct gk20a *g);
211 void (*isr)(struct gk20a *g);
212 u32 (*cbc_fix_config)(struct gk20a *g, int base);
213 void (*flush)(struct gk20a *g);
214 void (*intr_en_illegal_compstat)(struct gk20a *g, bool enable);
215 bool (*pri_is_ltc_addr)(struct gk20a *g, u32 addr);
216 bool (*is_ltcs_ltss_addr)(struct gk20a *g, u32 addr);
217 bool (*is_ltcn_ltss_addr)(struct gk20a *g, u32 addr);
218 void (*split_lts_broadcast_addr)(struct gk20a *g, u32 addr,
219 u32 *priv_addr_table,
220 u32 *priv_addr_table_index);
221 void (*split_ltc_broadcast_addr)(struct gk20a *g, u32 addr,
222 u32 *priv_addr_table,
223 u32 *priv_addr_table_index);
224 } ltc;
225 struct {
226 void (*isr_stall)(struct gk20a *g, u32 inst_id, u32 pri_base);
227 u32 (*isr_nonstall)(struct gk20a *g, u32 inst_id, u32 pri_base);
228 u32 (*get_num_pce)(struct gk20a *g);
229 void (*init_prod_values)(struct gk20a *g);
230 } ce2;
231 struct {
232 u32 (*get_patch_slots)(struct gk20a *g);
233 int (*init_fs_state)(struct gk20a *g);
234 int (*init_preemption_state)(struct gk20a *g);
235 void (*access_smpc_reg)(struct gk20a *g, u32 quad, u32 offset);
236 void (*bundle_cb_defaults)(struct gk20a *g);
237 void (*cb_size_default)(struct gk20a *g);
238 int (*calc_global_ctx_buffer_size)(struct gk20a *g);
239 void (*commit_global_attrib_cb)(struct gk20a *g,
240 struct nvgpu_gr_ctx *ch_ctx,
241 u64 addr, bool patch);
242 void (*commit_global_bundle_cb)(struct gk20a *g,
243 struct nvgpu_gr_ctx *ch_ctx,
244 u64 addr, u64 size, bool patch);
245 int (*commit_global_cb_manager)(struct gk20a *g,
246 struct channel_gk20a *ch,
247 bool patch);
248 void (*commit_global_pagepool)(struct gk20a *g,
249 struct nvgpu_gr_ctx *ch_ctx,
250 u64 addr, u32 size, bool patch);
251 void (*init_gpc_mmu)(struct gk20a *g);
252 int (*handle_sw_method)(struct gk20a *g, u32 addr,
253 u32 class_num, u32 offset, u32 data);
254 void (*set_alpha_circular_buffer_size)(struct gk20a *g,
255 u32 data);
256 void (*set_circular_buffer_size)(struct gk20a *g, u32 data);
257 void (*set_bes_crop_debug3)(struct gk20a *g, u32 data);
258 void (*set_bes_crop_debug4)(struct gk20a *g, u32 data);
259 void (*enable_hww_exceptions)(struct gk20a *g);
260 bool (*is_valid_class)(struct gk20a *g, u32 class_num);
261 bool (*is_valid_gfx_class)(struct gk20a *g, u32 class_num);
262 bool (*is_valid_compute_class)(struct gk20a *g, u32 class_num);
263 void (*get_sm_dsm_perf_regs)(struct gk20a *g,
264 u32 *num_sm_dsm_perf_regs,
265 u32 **sm_dsm_perf_regs,
266 u32 *perf_register_stride);
267 void (*get_sm_dsm_perf_ctrl_regs)(struct gk20a *g,
268 u32 *num_sm_dsm_perf_regs,
269 u32 **sm_dsm_perf_regs,
270 u32 *perf_register_stride);
271 void (*get_ovr_perf_regs)(struct gk20a *g,
272 u32 *num_ovr_perf_regs,
273 u32 **ovr_perf_regsr);
274 void (*set_hww_esr_report_mask)(struct gk20a *g);
275 int (*setup_alpha_beta_tables)(struct gk20a *g,
276 struct gr_gk20a *gr);
277 int (*falcon_load_ucode)(struct gk20a *g,
278 u64 addr_base,
279 struct gk20a_ctxsw_ucode_segments *segments,
280 u32 reg_offset);
281 int (*load_ctxsw_ucode)(struct gk20a *g);
282 u32 (*get_gpc_mask)(struct gk20a *g);
283 u32 (*get_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
284 void (*set_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
285 int (*alloc_obj_ctx)(struct channel_gk20a *c,
286 u32 class_num, u32 flags);
287 int (*bind_ctxsw_zcull)(struct gk20a *g, struct gr_gk20a *gr,
288 struct channel_gk20a *c, u64 zcull_va,
289 u32 mode);
290 int (*get_zcull_info)(struct gk20a *g, struct gr_gk20a *gr,
291 struct gr_zcull_info *zcull_params);
292 int (*decode_egpc_addr)(struct gk20a *g,
293 u32 addr, enum ctxsw_addr_type *addr_type,
294 u32 *gpc_num, u32 *tpc_num, u32 *broadcast_flags);
295 void (*egpc_etpc_priv_addr_table)(struct gk20a *g, u32 addr,
296 u32 gpc, u32 tpc, u32 broadcast_flags,
297 u32 *priv_addr_table,
298 u32 *priv_addr_table_index);
299 bool (*is_tpc_addr)(struct gk20a *g, u32 addr);
300 bool (*is_egpc_addr)(struct gk20a *g, u32 addr);
301 bool (*is_etpc_addr)(struct gk20a *g, u32 addr);
302 void (*get_egpc_etpc_num)(struct gk20a *g, u32 addr,
303 u32 *gpc_num, u32 *tpc_num);
304 u32 (*get_tpc_num)(struct gk20a *g, u32 addr);
305 u32 (*get_egpc_base)(struct gk20a *g);
306 void (*detect_sm_arch)(struct gk20a *g);
307 int (*add_zbc_color)(struct gk20a *g, struct gr_gk20a *gr,
308 struct zbc_entry *color_val, u32 index);
309 int (*add_zbc_depth)(struct gk20a *g, struct gr_gk20a *gr,
310 struct zbc_entry *depth_val, u32 index);
311 int (*add_zbc_s)(struct gk20a *g, struct gr_gk20a *gr,
312 struct zbc_entry *s_val, u32 index);
313 int (*zbc_set_table)(struct gk20a *g, struct gr_gk20a *gr,
314 struct zbc_entry *zbc_val);
315 int (*zbc_query_table)(struct gk20a *g, struct gr_gk20a *gr,
316 struct zbc_query_params *query_params);
317 int (*zbc_s_query_table)(struct gk20a *g, struct gr_gk20a *gr,
318 struct zbc_query_params *query_params);
319 int (*load_zbc_s_default_tbl)(struct gk20a *g,
320 struct gr_gk20a *gr);
321 int (*load_zbc_s_tbl)(struct gk20a *g,
322 struct gr_gk20a *gr);
323 void (*pmu_save_zbc)(struct gk20a *g, u32 entries);
324 int (*add_zbc)(struct gk20a *g, struct gr_gk20a *gr,
325 struct zbc_entry *zbc_val);
326 bool (*add_zbc_type_s)(struct gk20a *g, struct gr_gk20a *gr,
327 struct zbc_entry *zbc_val, int *ret_val);
328 u32 (*pagepool_default_size)(struct gk20a *g);
329 int (*init_ctx_state)(struct gk20a *g);
330 int (*alloc_gr_ctx)(struct gk20a *g,
331 struct nvgpu_gr_ctx *gr_ctx, struct vm_gk20a *vm,
332 u32 class, u32 padding);
333 void (*free_gr_ctx)(struct gk20a *g,
334 struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx);
335 void (*powergate_tpc)(struct gk20a *g);
336 void (*update_ctxsw_preemption_mode)(struct gk20a *g,
337 struct channel_gk20a *c,
338 struct nvgpu_mem *mem);
339 int (*update_smpc_ctxsw_mode)(struct gk20a *g,
340 struct channel_gk20a *c,
341 bool enable);
342 u32 (*get_hw_accessor_stream_out_mode)(void);
343 int (*update_hwpm_ctxsw_mode)(struct gk20a *g,
344 struct channel_gk20a *c,
345 u64 gpu_va,
346 u32 mode);
347 void (*init_hwpm_pmm_register)(struct gk20a *g);
348 void (*get_num_hwpm_perfmon)(struct gk20a *g, u32 *num_sys_perfmon,
349 u32 *num_fbp_perfmon, u32 *num_gpc_perfmon);
350 void (*set_pmm_register)(struct gk20a *g, u32 offset, u32 val,
351 u32 num_chiplets, u32 num_perfmons);
352 int (*dump_gr_regs)(struct gk20a *g,
353 struct gk20a_debug_output *o);
354 int (*update_pc_sampling)(struct channel_gk20a *ch,
355 bool enable);
356 u32 (*get_max_fbps_count)(struct gk20a *g);
357 u32 (*get_fbp_en_mask)(struct gk20a *g);
358 u32 (*get_max_ltc_per_fbp)(struct gk20a *g);
359 u32 (*get_max_lts_per_ltc)(struct gk20a *g);
360 u32* (*get_rop_l2_en_mask)(struct gk20a *g);
361 void (*init_sm_dsm_reg_info)(void);
362 void (*init_ovr_sm_dsm_perf)(void);
363 int (*wait_empty)(struct gk20a *g, unsigned long duration_ms,
364 u32 expect_delay);
365 void (*init_cyclestats)(struct gk20a *g);
366 void (*enable_cde_in_fecs)(struct gk20a *g,
367 struct nvgpu_mem *mem);
368 int (*set_sm_debug_mode)(struct gk20a *g, struct channel_gk20a *ch,
369 u64 sms, bool enable);
370 void (*bpt_reg_info)(struct gk20a *g,
371 struct nvgpu_warpstate *w_state);
372 void (*get_access_map)(struct gk20a *g,
373 u32 **whitelist, int *num_entries);
374 int (*handle_fecs_error)(struct gk20a *g,
375 struct channel_gk20a *ch,
376 struct gr_gk20a_isr_data *isr_data);
377 int (*pre_process_sm_exception)(struct gk20a *g,
378 u32 gpc, u32 tpc, u32 sm, u32 global_esr, u32 warp_esr,
379 bool sm_debugger_attached,
380 struct channel_gk20a *fault_ch,
381 bool *early_exit, bool *ignore_debugger);
382 u32 (*get_sm_hww_warp_esr)(struct gk20a *g,
383 u32 gpc, u32 tpc, u32 sm);
384 u32 (*get_sm_hww_global_esr)(struct gk20a *g,
385 u32 gpc, u32 tpc, u32 sm);
386 u32 (*get_sm_no_lock_down_hww_global_esr_mask)(struct gk20a *g);
387 int (*lock_down_sm)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm,
388 u32 global_esr_mask, bool check_errors);
389 int (*wait_for_sm_lock_down)(struct gk20a *g, u32 gpc, u32 tpc,
390 u32 sm, u32 global_esr_mask, bool check_errors);
391 void (*clear_sm_hww)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm,
392 u32 global_esr);
393 void (*get_esr_sm_sel)(struct gk20a *g, u32 gpc, u32 tpc,
394 u32 *esr_sm_sel);
395 int (*handle_tpc_sm_ecc_exception)(struct gk20a *g,
396 u32 gpc, u32 tpc,
397 bool *post_event, struct channel_gk20a *fault_ch,
398 u32 *hww_global_esr);
399 int (*handle_sm_exception)(struct gk20a *g,
400 u32 gpc, u32 tpc, u32 sm,
401 bool *post_event, struct channel_gk20a *fault_ch,
402 u32 *hww_global_esr);
403 int (*handle_gcc_exception)(struct gk20a *g, u32 gpc, u32 tpc,
404 bool *post_event, struct channel_gk20a *fault_ch,
405 u32 *hww_global_esr);
406 int (*handle_tex_exception)(struct gk20a *g, u32 gpc, u32 tpc,
407 bool *post_event);
408 int (*handle_tpc_mpc_exception)(struct gk20a *g,
409 u32 gpc, u32 tpc, bool *post_event);
410 int (*handle_gpc_gpccs_exception)(struct gk20a *g, u32 gpc,
411 u32 gpc_exception);
412 int (*handle_gpc_gpcmmu_exception)(struct gk20a *g, u32 gpc,
413 u32 gpc_exception);
414 void (*enable_gpc_exceptions)(struct gk20a *g);
415 void (*enable_exceptions)(struct gk20a *g);
416 int (*init_ecc)(struct gk20a *g);
417 u32 (*get_lrf_tex_ltc_dram_override)(struct gk20a *g);
418 int (*record_sm_error_state)(struct gk20a *g, u32 gpc, u32 tpc,
419 u32 sm, struct channel_gk20a *fault_ch);
420 int (*clear_sm_error_state)(struct gk20a *g,
421 struct channel_gk20a *ch, u32 sm_id);
422 int (*suspend_contexts)(struct gk20a *g,
423 struct dbg_session_gk20a *dbg_s,
424 int *ctx_resident_ch_fd);
425 int (*resume_contexts)(struct gk20a *g,
426 struct dbg_session_gk20a *dbg_s,
427 int *ctx_resident_ch_fd);
428 int (*set_preemption_mode)(struct channel_gk20a *ch,
429 u32 graphics_preempt_mode,
430 u32 compute_preempt_mode);
431 int (*get_preemption_mode_flags)(struct gk20a *g,
432 struct nvgpu_preemption_modes_rec *preemption_modes_rec);
433 int (*set_ctxsw_preemption_mode)(struct gk20a *g,
434 struct nvgpu_gr_ctx *gr_ctx,
435 struct vm_gk20a *vm, u32 class,
436 u32 graphics_preempt_mode,
437 u32 compute_preempt_mode);
438 int (*set_boosted_ctx)(struct channel_gk20a *ch, bool boost);
439 void (*update_boosted_ctx)(struct gk20a *g,
440 struct nvgpu_mem *mem,
441 struct nvgpu_gr_ctx *gr_ctx);
442 int (*init_sm_id_table)(struct gk20a *g);
443 int (*load_smid_config)(struct gk20a *g);
444 void (*program_sm_id_numbering)(struct gk20a *g,
445 u32 gpc, u32 tpc, u32 smid);
446 void (*program_active_tpc_counts)(struct gk20a *g, u32 gpc);
447 int (*setup_rop_mapping)(struct gk20a *g, struct gr_gk20a *gr);
448 int (*init_sw_veid_bundle)(struct gk20a *g);
449 void (*program_zcull_mapping)(struct gk20a *g,
450 u32 zcull_alloc_num, u32 *zcull_map_tiles);
451 int (*commit_global_timeslice)(struct gk20a *g,
452 struct channel_gk20a *c);
453 int (*commit_inst)(struct channel_gk20a *c, u64 gpu_va);
454 void (*write_zcull_ptr)(struct gk20a *g,
455 struct nvgpu_mem *mem, u64 gpu_va);
456 void (*write_pm_ptr)(struct gk20a *g,
457 struct nvgpu_mem *mem, u64 gpu_va);
458 void (*set_preemption_buffer_va)(struct gk20a *g,
459 struct nvgpu_mem *mem, u64 gpu_va);
460 void (*load_tpc_mask)(struct gk20a *g);
461 int (*trigger_suspend)(struct gk20a *g);
462 int (*wait_for_pause)(struct gk20a *g, struct nvgpu_warpstate *w_state);
463 int (*resume_from_pause)(struct gk20a *g);
464 int (*clear_sm_errors)(struct gk20a *g);
465 u32 (*tpc_enabled_exceptions)(struct gk20a *g);
466 int (*set_czf_bypass)(struct gk20a *g,
467 struct channel_gk20a *ch);
468 void (*init_czf_bypass)(struct gk20a *g);
469 bool (*sm_debugger_attached)(struct gk20a *g);
470 void (*suspend_single_sm)(struct gk20a *g,
471 u32 gpc, u32 tpc, u32 sm,
472 u32 global_esr_mask, bool check_errors);
473 void (*suspend_all_sms)(struct gk20a *g,
474 u32 global_esr_mask, bool check_errors);
475 void (*resume_single_sm)(struct gk20a *g,
476 u32 gpc, u32 tpc, u32 sm);
477 void (*resume_all_sms)(struct gk20a *g);
478 void (*disable_rd_coalesce)(struct gk20a *g);
479 void (*init_ctxsw_hdr_data)(struct gk20a *g,
480 struct nvgpu_mem *mem);
481 void (*init_gfxp_wfi_timeout_count)(struct gk20a *g);
482 unsigned long (*get_max_gfxp_wfi_timeout_count)
483 (struct gk20a *g);
484 void (*ecc_init_scrub_reg)(struct gk20a *g);
485 u32 (*get_gpcs_swdx_dss_zbc_c_format_reg)(struct gk20a *g);
486 u32 (*get_gpcs_swdx_dss_zbc_z_format_reg)(struct gk20a *g);
487 void (*dump_ctxsw_stats)(struct gk20a *g, struct vm_gk20a *vm,
488 struct nvgpu_gr_ctx *gr_ctx);
489 void (*fecs_host_int_enable)(struct gk20a *g);
490 int (*handle_ssync_hww)(struct gk20a *g);
491 int (*handle_notify_pending)(struct gk20a *g,
492 struct gr_gk20a_isr_data *isr_data);
493 int (*handle_semaphore_pending)(struct gk20a *g,
494 struct gr_gk20a_isr_data *isr_data);
495 int (*add_ctxsw_reg_pm_fbpa)(struct gk20a *g,
496 struct ctxsw_buf_offset_map_entry *map,
497 struct aiv_list_gk20a *regs,
498 u32 *count, u32 *offset,
499 u32 max_cnt, u32 base,
500 u32 num_fbpas, u32 stride, u32 mask);
501 int (*add_ctxsw_reg_perf_pma)(struct ctxsw_buf_offset_map_entry *map,
502 struct aiv_list_gk20a *regs,
503 u32 *count, u32 *offset,
504 u32 max_cnt, u32 base, u32 mask);
505 int (*decode_priv_addr)(struct gk20a *g, u32 addr,
506 enum ctxsw_addr_type *addr_type,
507 u32 *gpc_num, u32 *tpc_num,
508 u32 *ppc_num, u32 *be_num,
509 u32 *broadcast_flags);
510 int (*create_priv_addr_table)(struct gk20a *g,
511 u32 addr,
512 u32 *priv_addr_table,
513 u32 *num_registers);
514 u32 (*get_pmm_per_chiplet_offset)(void);
515 void (*split_fbpa_broadcast_addr)(struct gk20a *g, u32 addr,
516 u32 num_fbpas,
517 u32 *priv_addr_table,
518 u32 *priv_addr_table_index);
519 u32 (*fecs_ctxsw_mailbox_size)(void);
520 u32 (*gpc0_gpccs_ctxsw_mailbox_size)(void);
521 int (*init_sw_bundle64)(struct gk20a *g);
522 int (*alloc_global_ctx_buffers)(struct gk20a *g);
523 int (*map_global_ctx_buffers)(struct gk20a *g,
524 struct channel_gk20a *c);
525 int (*commit_global_ctx_buffers)(struct gk20a *g,
526 struct channel_gk20a *c, bool patch);
527 u32 (*get_nonpes_aware_tpc)(struct gk20a *g, u32 gpc, u32 tpc);
528 int (*get_offset_in_gpccs_segment)(struct gk20a *g,
529 enum ctxsw_addr_type addr_type, u32 num_tpcs,
530 u32 num_ppcs, u32 reg_list_ppc_count,
531 u32 *__offset_in_segment);
532 void (*set_debug_mode)(struct gk20a *g, bool enable);
533 int (*set_mmu_debug_mode)(struct gk20a *g,
534 struct channel_gk20a *ch, bool enable);
535 int (*set_fecs_watchdog_timeout)(struct gk20a *g);
536 } gr;
537 struct {
538 void (*init_hw)(struct gk20a *g);
539 void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr);
540 void (*init_fs_state)(struct gk20a *g);
541 void (*init_uncompressed_kind_map)(struct gk20a *g);
542 void (*init_kind_attr)(struct gk20a *g);
543 void (*set_mmu_page_size)(struct gk20a *g);
544 bool (*set_use_full_comp_tag_line)(struct gk20a *g);
545 u32 (*mmu_ctrl)(struct gk20a *g);
546 u32 (*mmu_debug_ctrl)(struct gk20a *g);
547 u32 (*mmu_debug_wr)(struct gk20a *g);
548 u32 (*mmu_debug_rd)(struct gk20a *g);
549
550 /*
551 * Compression tag line coverage. When mapping a compressible
552 * buffer, ctagline is increased when the virtual address
553 * crosses over the compression page boundary.
554 */
555 unsigned int (*compression_page_size)(struct gk20a *g);
556
557 /*
558 * Minimum page size that can be used for compressible kinds.
559 */
560 unsigned int (*compressible_page_size)(struct gk20a *g);
561
562 /*
563 * Compressible kind mappings: Mask for the virtual and physical
564 * address bits that must match.
565 */
566 u32 (*compression_align_mask)(struct gk20a *g);
567
568 void (*dump_vpr_info)(struct gk20a *g);
569 void (*dump_wpr_info)(struct gk20a *g);
570 int (*vpr_info_fetch)(struct gk20a *g);
571 void (*read_wpr_info)(struct gk20a *g,
572 struct wpr_carveout_info *inf);
573 bool (*is_debug_mode_enabled)(struct gk20a *g);
574 void (*set_debug_mode)(struct gk20a *g, bool enable);
575 void (*set_mmu_debug_mode)(struct gk20a *g, bool enable);
576 int (*tlb_invalidate)(struct gk20a *g, struct nvgpu_mem *pdb);
577 void (*hub_isr)(struct gk20a *g);
578 void (*handle_replayable_fault)(struct gk20a *g);
579 int (*mem_unlock)(struct gk20a *g);
580 int (*init_nvlink)(struct gk20a *g);
581 int (*enable_nvlink)(struct gk20a *g);
582 void (*enable_hub_intr)(struct gk20a *g);
583 void (*disable_hub_intr)(struct gk20a *g);
584 int (*init_fbpa)(struct gk20a *g);
585 void (*handle_fbpa_intr)(struct gk20a *g, u32 fbpa_id);
586 void (*write_mmu_fault_buffer_lo_hi)(struct gk20a *g, u32 index,
587 u32 addr_lo, u32 addr_hi);
588 void (*write_mmu_fault_buffer_get)(struct gk20a *g, u32 index,
589 u32 reg_val);
590 void (*write_mmu_fault_buffer_size)(struct gk20a *g, u32 index,
591 u32 reg_val);
592 void (*write_mmu_fault_status)(struct gk20a *g, u32 reg_val);
593 u32 (*read_mmu_fault_buffer_get)(struct gk20a *g, u32 index);
594 u32 (*read_mmu_fault_buffer_put)(struct gk20a *g, u32 index);
595 u32 (*read_mmu_fault_buffer_size)(struct gk20a *g, u32 index);
596 void (*read_mmu_fault_addr_lo_hi)(struct gk20a *g,
597 u32 *addr_lo, u32 *addr_hi);
598 void (*read_mmu_fault_inst_lo_hi)(struct gk20a *g,
599 u32 *inst_lo, u32 *inst_hi);
600 u32 (*read_mmu_fault_info)(struct gk20a *g);
601 u32 (*read_mmu_fault_status)(struct gk20a *g);
602 int (*mmu_invalidate_replay)(struct gk20a *g,
603 u32 invalidate_replay_val);
604 bool (*mmu_fault_pending)(struct gk20a *g);
605 bool (*is_fault_buf_enabled)(struct gk20a *g, u32 index);
606 void (*fault_buf_set_state_hw)(struct gk20a *g,
607 u32 index, u32 state);
608 void (*fault_buf_configure_hw)(struct gk20a *g, u32 index);
609 size_t (*get_vidmem_size)(struct gk20a *g);
610 int (*apply_pdb_cache_war)(struct gk20a *g);
611 } fb;
612 struct {
613 void (*slcg_bus_load_gating_prod)(struct gk20a *g, bool prod);
614 void (*slcg_ce2_load_gating_prod)(struct gk20a *g, bool prod);
615 void (*slcg_chiplet_load_gating_prod)(struct gk20a *g, bool prod);
616 void (*slcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod);
617 void (*slcg_fb_load_gating_prod)(struct gk20a *g, bool prod);
618 void (*slcg_fifo_load_gating_prod)(struct gk20a *g, bool prod);
619 void (*slcg_gr_load_gating_prod)(struct gk20a *g, bool prod);
620 void (*slcg_ltc_load_gating_prod)(struct gk20a *g, bool prod);
621 void (*slcg_perf_load_gating_prod)(struct gk20a *g, bool prod);
622 void (*slcg_priring_load_gating_prod)(struct gk20a *g, bool prod);
623 void (*slcg_pmu_load_gating_prod)(struct gk20a *g, bool prod);
624 void (*slcg_therm_load_gating_prod)(struct gk20a *g, bool prod);
625 void (*slcg_xbar_load_gating_prod)(struct gk20a *g, bool prod);
626 void (*slcg_hshub_load_gating_prod)(struct gk20a *g, bool prod);
627 void (*slcg_acb_load_gating_prod)(struct gk20a *g, bool prod);
628 void (*blcg_bus_load_gating_prod)(struct gk20a *g, bool prod);
629 void (*blcg_ce_load_gating_prod)(struct gk20a *g, bool prod);
630 void (*blcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod);
631 void (*blcg_fb_load_gating_prod)(struct gk20a *g, bool prod);
632 void (*blcg_fifo_load_gating_prod)(struct gk20a *g, bool prod);
633 void (*blcg_gr_load_gating_prod)(struct gk20a *g, bool prod);
634 void (*blcg_ltc_load_gating_prod)(struct gk20a *g, bool prod);
635 void (*blcg_pwr_csb_load_gating_prod)(struct gk20a *g, bool prod);
636 void (*blcg_pmu_load_gating_prod)(struct gk20a *g, bool prod);
637 void (*blcg_xbar_load_gating_prod)(struct gk20a *g, bool prod);
638 void (*blcg_hshub_load_gating_prod)(struct gk20a *g, bool prod);
639 void (*pg_gr_load_gating_prod)(struct gk20a *g, bool prod);
640 } clock_gating;
641 struct {
642 void (*post_events)(struct channel_gk20a *ch);
643 } debugger;
644 struct {
645 int (*setup_sw)(struct gk20a *g);
646 int (*init_fifo_setup_hw)(struct gk20a *g);
647 void (*bind_channel)(struct channel_gk20a *ch_gk20a);
648 void (*unbind_channel)(struct channel_gk20a *ch_gk20a);
649 void (*disable_channel)(struct channel_gk20a *ch);
650 void (*enable_channel)(struct channel_gk20a *ch);
651 int (*alloc_inst)(struct gk20a *g, struct channel_gk20a *ch);
652 void (*free_inst)(struct gk20a *g, struct channel_gk20a *ch);
653 int (*setup_ramfc)(struct channel_gk20a *c, u64 gpfifo_base,
654 u32 gpfifo_entries,
655 unsigned long acquire_timeout,
656 u32 flags);
657 int (*resetup_ramfc)(struct channel_gk20a *c);
658 int (*preempt_channel)(struct gk20a *g, struct channel_gk20a *ch);
659 int (*preempt_tsg)(struct gk20a *g, struct tsg_gk20a *tsg);
660 int (*enable_tsg)(struct tsg_gk20a *tsg);
661 int (*disable_tsg)(struct tsg_gk20a *tsg);
662 int (*tsg_verify_channel_status)(struct channel_gk20a *ch);
663 void (*tsg_verify_status_ctx_reload)(struct channel_gk20a *ch);
664 void (*tsg_verify_status_faulted)(struct channel_gk20a *ch);
665 int (*reschedule_runlist)(struct channel_gk20a *ch,
666 bool preempt_next);
667 int (*update_runlist)(struct gk20a *g, u32 runlist_id,
668 u32 chid, bool add,
669 bool wait_for_finish);
670 void (*trigger_mmu_fault)(struct gk20a *g,
671 unsigned long engine_ids);
672 void (*get_mmu_fault_info)(struct gk20a *g, u32 mmu_fault_id,
673 struct mmu_fault_info *mmfault);
674 void (*get_mmu_fault_desc)(struct mmu_fault_info *mmfault);
675 void (*get_mmu_fault_client_desc)(
676 struct mmu_fault_info *mmfault);
677 void (*get_mmu_fault_gpc_desc)(struct mmu_fault_info *mmfault);
678 void (*apply_pb_timeout)(struct gk20a *g);
679 void (*apply_ctxsw_timeout_intr)(struct gk20a *g);
680 int (*wait_engine_idle)(struct gk20a *g);
681 u32 (*get_num_fifos)(struct gk20a *g);
682 u32 (*get_pbdma_signature)(struct gk20a *g);
683 int (*set_runlist_interleave)(struct gk20a *g, u32 id,
684 u32 runlist_id,
685 u32 new_level);
686 int (*tsg_set_timeslice)(struct tsg_gk20a *tsg, u32 timeslice);
687 u32 (*default_timeslice_us)(struct gk20a *);
688 int (*force_reset_ch)(struct channel_gk20a *ch,
689 u32 err_code, bool verbose);
690 int (*engine_enum_from_type)(struct gk20a *g, u32 engine_type,
691 u32 *inst_id);
692 void (*device_info_data_parse)(struct gk20a *g,
693 u32 table_entry, u32 *inst_id,
694 u32 *pri_base, u32 *fault_id);
695 u32 (*device_info_fault_id)(u32 table_entry);
696 int (*tsg_bind_channel)(struct tsg_gk20a *tsg,
697 struct channel_gk20a *ch);
698 int (*tsg_unbind_channel)(struct channel_gk20a *ch);
699 int (*tsg_open)(struct tsg_gk20a *tsg);
700 void (*tsg_release)(struct tsg_gk20a *tsg);
701 u32 (*eng_runlist_base_size)(void);
702 int (*init_engine_info)(struct fifo_gk20a *f);
703 u32 (*runlist_entry_size)(void);
704 void (*get_tsg_runlist_entry)(struct tsg_gk20a *tsg,
705 u32 *runlist);
706 void (*get_ch_runlist_entry)(struct channel_gk20a *ch,
707 u32 *runlist);
708 u32 (*userd_gp_get)(struct gk20a *g, struct channel_gk20a *ch);
709 void (*userd_gp_put)(struct gk20a *g, struct channel_gk20a *ch);
710 u64 (*userd_pb_get)(struct gk20a *g, struct channel_gk20a *ch);
711 void (*free_channel_ctx_header)(struct channel_gk20a *ch);
712 bool (*is_fault_engine_subid_gpc)(struct gk20a *g,
713 u32 engine_subid);
714 void (*dump_pbdma_status)(struct gk20a *g,
715 struct gk20a_debug_output *o);
716 void (*dump_eng_status)(struct gk20a *g,
717 struct gk20a_debug_output *o);
718 void (*dump_channel_status_ramfc)(struct gk20a *g,
719 struct gk20a_debug_output *o, u32 chid,
720 struct ch_state *ch_state);
721 u32 (*intr_0_error_mask)(struct gk20a *g);
722 int (*is_preempt_pending)(struct gk20a *g, u32 id,
723 unsigned int id_type, bool preempt_retries_left);
724 void (*init_pbdma_intr_descs)(struct fifo_gk20a *f);
725 int (*reset_enable_hw)(struct gk20a *g);
726 int (*setup_userd)(struct channel_gk20a *c);
727 u32 (*pbdma_acquire_val)(u64 timeout);
728 void (*teardown_ch_tsg)(struct gk20a *g, u32 act_eng_bitmask,
729 u32 id, unsigned int id_type, unsigned int rc_type,
730 struct mmu_fault_info *mmfault);
731 void (*teardown_mask_intr)(struct gk20a *g);
732 void (*teardown_unmask_intr)(struct gk20a *g);
733 bool (*handle_sched_error)(struct gk20a *g);
734 bool (*handle_ctxsw_timeout)(struct gk20a *g, u32 fifo_intr);
735 unsigned int (*handle_pbdma_intr_0)(struct gk20a *g,
736 u32 pbdma_id, u32 pbdma_intr_0,
737 u32 *handled, u32 *error_notifier);
738 unsigned int (*handle_pbdma_intr_1)(struct gk20a *g,
739 u32 pbdma_id, u32 pbdma_intr_1,
740 u32 *handled, u32 *error_notifier);
741 void (*init_eng_method_buffers)(struct gk20a *g,
742 struct tsg_gk20a *tsg);
743 void (*deinit_eng_method_buffers)(struct gk20a *g,
744 struct tsg_gk20a *tsg);
745 u32 (*get_preempt_timeout)(struct gk20a *g);
746 void (*post_event_id)(struct tsg_gk20a *tsg, int event_id);
747 void (*ch_abort_clean_up)(struct channel_gk20a *ch);
748 bool (*check_tsg_ctxsw_timeout)(struct tsg_gk20a *tsg,
749 bool *verbose, u32 *ms);
750 bool (*check_ch_ctxsw_timeout)(struct channel_gk20a *ch,
751 bool *verbose, u32 *ms);
752 int (*channel_suspend)(struct gk20a *g);
753 int (*channel_resume)(struct gk20a *g);
754 void (*set_error_notifier)(struct channel_gk20a *ch, u32 error);
755#ifdef CONFIG_TEGRA_GK20A_NVHOST
756 int (*alloc_syncpt_buf)(struct channel_gk20a *c,
757 u32 syncpt_id, struct nvgpu_mem *syncpt_buf);
758 void (*free_syncpt_buf)(struct channel_gk20a *c,
759 struct nvgpu_mem *syncpt_buf);
760 void (*add_syncpt_wait_cmd)(struct gk20a *g,
761 struct priv_cmd_entry *cmd, u32 off,
762 u32 id, u32 thresh, u64 gpu_va);
763 u32 (*get_syncpt_wait_cmd_size)(void);
764 void (*add_syncpt_incr_cmd)(struct gk20a *g,
765 bool wfi_cmd, struct priv_cmd_entry *cmd,
766 u32 id, u64 gpu_va);
767 u32 (*get_syncpt_incr_cmd_size)(bool wfi_cmd);
768 int (*get_sync_ro_map)(struct vm_gk20a *vm,
769 u64 *base_gpuva, u32 *sync_size);
770 u32 (*get_syncpt_incr_per_release)(void);
771#endif
772 void (*runlist_hw_submit)(struct gk20a *g, u32 runlist_id,
773 u32 count, u32 buffer_index);
774 int (*runlist_wait_pending)(struct gk20a *g, u32 runlist_id);
775 void (*ring_channel_doorbell)(struct channel_gk20a *c);
776 u64 (*usermode_base)(struct gk20a *g);
777 u32 (*get_sema_wait_cmd_size)(void);
778 u32 (*get_sema_incr_cmd_size)(void);
779 void (*add_sema_cmd)(struct gk20a *g,
780 struct nvgpu_semaphore *s, u64 sema_va,
781 struct priv_cmd_entry *cmd,
782 u32 off, bool acquire, bool wfi);
783 int (*init_pdb_cache_war)(struct gk20a *g);
784 void (*deinit_pdb_cache_war)(struct gk20a *g);
785 } fifo;
786 struct pmu_v {
787 u32 (*get_pmu_cmdline_args_size)(struct nvgpu_pmu *pmu);
788 void (*set_pmu_cmdline_args_cpu_freq)(struct nvgpu_pmu *pmu,
789 u32 freq);
790 void (*set_pmu_cmdline_args_trace_size)(struct nvgpu_pmu *pmu,
791 u32 size);
792 void (*set_pmu_cmdline_args_trace_dma_base)(
793 struct nvgpu_pmu *pmu);
794 void (*config_pmu_cmdline_args_super_surface)(
795 struct nvgpu_pmu *pmu);
796 void (*set_pmu_cmdline_args_trace_dma_idx)(
797 struct nvgpu_pmu *pmu, u32 idx);
798 void * (*get_pmu_cmdline_args_ptr)(struct nvgpu_pmu *pmu);
799 u32 (*get_pmu_allocation_struct_size)(struct nvgpu_pmu *pmu);
800 void (*set_pmu_allocation_ptr)(struct nvgpu_pmu *pmu,
801 void **pmu_alloc_ptr, void *assign_ptr);
802 void (*pmu_allocation_set_dmem_size)(struct nvgpu_pmu *pmu,
803 void *pmu_alloc_ptr, u16 size);
804 u16 (*pmu_allocation_get_dmem_size)(struct nvgpu_pmu *pmu,
805 void *pmu_alloc_ptr);
806 u32 (*pmu_allocation_get_dmem_offset)(struct nvgpu_pmu *pmu,
807 void *pmu_alloc_ptr);
808 u32 * (*pmu_allocation_get_dmem_offset_addr)(
809 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr);
810 void (*pmu_allocation_set_dmem_offset)(struct nvgpu_pmu *pmu,
811 void *pmu_alloc_ptr, u32 offset);
812 void * (*pmu_allocation_get_fb_addr)(
813 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr);
814 u32 (*pmu_allocation_get_fb_size)(
815 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr);
816 void (*get_pmu_init_msg_pmu_queue_params)(
817 struct nvgpu_falcon_queue *queue, u32 id,
818 void *pmu_init_msg);
819 void *(*get_pmu_msg_pmu_init_msg_ptr)(
820 struct pmu_init_msg *init);
821 u16 (*get_pmu_init_msg_pmu_sw_mg_off)(
822 union pmu_init_msg_pmu *init_msg);
823 u16 (*get_pmu_init_msg_pmu_sw_mg_size)(
824 union pmu_init_msg_pmu *init_msg);
825 u32 (*get_pmu_perfmon_cmd_start_size)(void);
826 int (*get_perfmon_cmd_start_offsetofvar)(
827 enum pmu_perfmon_cmd_start_fields field);
828 void (*perfmon_start_set_cmd_type)(struct pmu_perfmon_cmd *pc,
829 u8 value);
830 void (*perfmon_start_set_group_id)(struct pmu_perfmon_cmd *pc,
831 u8 value);
832 void (*perfmon_start_set_state_id)(struct pmu_perfmon_cmd *pc,
833 u8 value);
834 void (*perfmon_start_set_flags)(struct pmu_perfmon_cmd *pc,
835 u8 value);
836 u8 (*perfmon_start_get_flags)(struct pmu_perfmon_cmd *pc);
837 u32 (*get_pmu_perfmon_cmd_init_size)(void);
838 int (*get_perfmon_cmd_init_offsetofvar)(
839 enum pmu_perfmon_cmd_start_fields field);
840 void (*perfmon_cmd_init_set_sample_buffer)(
841 struct pmu_perfmon_cmd *pc, u16 value);
842 void (*perfmon_cmd_init_set_dec_cnt)(
843 struct pmu_perfmon_cmd *pc, u8 value);
844 void (*perfmon_cmd_init_set_base_cnt_id)(
845 struct pmu_perfmon_cmd *pc, u8 value);
846 void (*perfmon_cmd_init_set_samp_period_us)(
847 struct pmu_perfmon_cmd *pc, u32 value);
848 void (*perfmon_cmd_init_set_num_cnt)(struct pmu_perfmon_cmd *pc,
849 u8 value);
850 void (*perfmon_cmd_init_set_mov_avg)(struct pmu_perfmon_cmd *pc,
851 u8 value);
852 void *(*get_pmu_seq_in_a_ptr)(
853 struct pmu_sequence *seq);
854 void *(*get_pmu_seq_out_a_ptr)(
855 struct pmu_sequence *seq);
856 void (*set_pmu_cmdline_args_secure_mode)(struct nvgpu_pmu *pmu,
857 u32 val);
858 u32 (*get_perfmon_cntr_sz)(struct nvgpu_pmu *pmu);
859 void * (*get_perfmon_cntr_ptr)(struct nvgpu_pmu *pmu);
860 void (*set_perfmon_cntr_ut)(struct nvgpu_pmu *pmu, u16 ut);
861 void (*set_perfmon_cntr_lt)(struct nvgpu_pmu *pmu, u16 lt);
862 void (*set_perfmon_cntr_valid)(struct nvgpu_pmu *pmu, u8 val);
863 void (*set_perfmon_cntr_index)(struct nvgpu_pmu *pmu, u8 val);
864 void (*set_perfmon_cntr_group_id)(struct nvgpu_pmu *pmu,
865 u8 gid);
866
867 u8 (*pg_cmd_eng_buf_load_size)(struct pmu_pg_cmd *pg);
868 void (*pg_cmd_eng_buf_load_set_cmd_type)(struct pmu_pg_cmd *pg,
869 u8 value);
870 void (*pg_cmd_eng_buf_load_set_engine_id)(struct pmu_pg_cmd *pg,
871 u8 value);
872 void (*pg_cmd_eng_buf_load_set_buf_idx)(struct pmu_pg_cmd *pg,
873 u8 value);
874 void (*pg_cmd_eng_buf_load_set_pad)(struct pmu_pg_cmd *pg,
875 u8 value);
876 void (*pg_cmd_eng_buf_load_set_buf_size)(struct pmu_pg_cmd *pg,
877 u16 value);
878 void (*pg_cmd_eng_buf_load_set_dma_base)(struct pmu_pg_cmd *pg,
879 u32 value);
880 void (*pg_cmd_eng_buf_load_set_dma_offset)(struct pmu_pg_cmd *pg,
881 u8 value);
882 void (*pg_cmd_eng_buf_load_set_dma_idx)(struct pmu_pg_cmd *pg,
883 u8 value);
884 struct {
885 int (*boardobjgrp_pmucmd_construct_impl)
886 (struct gk20a *g,
887 struct boardobjgrp *pboardobjgrp,
888 struct boardobjgrp_pmu_cmd *cmd, u8 id, u8 msgid,
889 u16 hdrsize, u16 entrysize, u16 fbsize, u32 ss_offset,
890 u8 rpc_func_id);
891 int (*boardobjgrp_pmuset_impl)(struct gk20a *g,
892 struct boardobjgrp *pboardobjgrp);
893 int (*boardobjgrp_pmugetstatus_impl)(struct gk20a *g,
894 struct boardobjgrp *pboardobjgrp,
895 struct boardobjgrpmask *mask);
896 int (*is_boardobjgrp_pmucmd_id_valid)(struct gk20a *g,
897 struct boardobjgrp *pboardobjgrp,
898 struct boardobjgrp_pmu_cmd *cmd);
899 } boardobj;
900 struct {
901 u32 (*volt_set_voltage)(struct gk20a *g,
902 u32 logic_voltage_uv, u32 sram_voltage_uv);
903 u32 (*volt_get_voltage)(struct gk20a *g,
904 u8 volt_domain, u32 *pvoltage_uv);
905 u32 (*volt_send_load_cmd_to_pmu)(struct gk20a *g);
906 } volt;
907 struct {
908 u32 (*get_vbios_clk_domain)(u32 vbios_domain);
909 u32 (*clk_avfs_get_vin_cal_data)(struct gk20a *g,
910 struct avfsvinobjs *pvinobjs,
911 struct vin_device_v20 *pvindev);
912 u32 (*clk_vf_change_inject_data_fill)(struct gk20a *g,
913 struct nv_pmu_clk_rpc *rpccall,
914 struct set_fll_clk *setfllclk);
915 u32 (*clk_set_boot_clk)(struct gk20a *g);
916 }clk;
917 } pmu_ver;
918 struct {
919 int (*get_netlist_name)(struct gk20a *g, int index, char *name);
920 bool (*is_fw_defined)(void);
921 } gr_ctx;
922#ifdef CONFIG_GK20A_CTXSW_TRACE
923 /*
924 * Currently only supported on Linux due to the extremely tight
925 * integration with Linux device driver structure (in particular
926 * mmap).
927 */
928 struct {
929 int (*init)(struct gk20a *g);
930 int (*max_entries)(struct gk20a *,
931 struct nvgpu_gpu_ctxsw_trace_filter *filter);
932 int (*flush)(struct gk20a *g);
933 int (*poll)(struct gk20a *g);
934 int (*enable)(struct gk20a *g);
935 int (*disable)(struct gk20a *g);
936 bool (*is_enabled)(struct gk20a *g);
937 int (*reset)(struct gk20a *g);
938 int (*bind_channel)(struct gk20a *g, struct channel_gk20a *ch);
939 int (*unbind_channel)(struct gk20a *g,
940 struct channel_gk20a *ch);
941 int (*deinit)(struct gk20a *g);
942 int (*alloc_user_buffer)(struct gk20a *g,
943 void **buf, size_t *size);
944 int (*free_user_buffer)(struct gk20a *g);
945 int (*mmap_user_buffer)(struct gk20a *g,
946 struct vm_area_struct *vma);
947 int (*set_filter)(struct gk20a *g,
948 struct nvgpu_gpu_ctxsw_trace_filter *filter);
949 } fecs_trace;
950#endif
951 struct {
952 bool (*support_sparse)(struct gk20a *g);
953 u64 (*gmmu_map)(struct vm_gk20a *vm,
954 u64 map_offset,
955 struct nvgpu_sgt *sgt,
956 u64 buffer_offset,
957 u64 size,
958 u32 pgsz_idx,
959 u8 kind_v,
960 u32 ctag_offset,
961 u32 flags,
962 enum gk20a_mem_rw_flag rw_flag,
963 bool clear_ctags,
964 bool sparse,
965 bool priv,
966 struct vm_gk20a_mapping_batch *batch,
967 enum nvgpu_aperture aperture);
968 void (*gmmu_unmap)(struct vm_gk20a *vm,
969 u64 vaddr,
970 u64 size,
971 u32 pgsz_idx,
972 bool va_allocated,
973 enum gk20a_mem_rw_flag rw_flag,
974 bool sparse,
975 struct vm_gk20a_mapping_batch *batch);
976 int (*vm_bind_channel)(struct vm_gk20a *vm,
977 struct channel_gk20a *ch);
978 int (*fb_flush)(struct gk20a *g);
979 void (*l2_invalidate)(struct gk20a *g);
980 void (*l2_flush)(struct gk20a *g, bool invalidate);
981 void (*cbc_clean)(struct gk20a *g);
982 void (*set_big_page_size)(struct gk20a *g,
983 struct nvgpu_mem *mem, int size);
984 u32 (*get_big_page_sizes)(void);
985 u32 (*get_default_big_page_size)(void);
986 u32 (*get_iommu_bit)(struct gk20a *g);
987 int (*init_mm_setup_hw)(struct gk20a *g);
988 bool (*is_bar1_supported)(struct gk20a *g);
989 int (*init_bar2_vm)(struct gk20a *g);
990 void (*remove_bar2_vm)(struct gk20a *g);
991 const struct gk20a_mmu_level *
992 (*get_mmu_levels)(struct gk20a *g, u32 big_page_size);
993 void (*init_pdb)(struct gk20a *g, struct nvgpu_mem *inst_block,
994 struct vm_gk20a *vm);
995 u64 (*gpu_phys_addr)(struct gk20a *g,
996 struct nvgpu_gmmu_attrs *attrs, u64 phys);
997 int (*alloc_inst_block)(struct gk20a *g,
998 struct nvgpu_mem *inst_block);
999 void (*init_inst_block)(struct nvgpu_mem *inst_block,
1000 struct vm_gk20a *vm, u32 big_page_size);
1001 bool (*mmu_fault_pending)(struct gk20a *g);
1002 void (*fault_info_mem_destroy)(struct gk20a *g);
1003 void (*mmu_fault_disable_hw)(struct gk20a *g);
1004 u32 (*get_kind_invalid)(void);
1005 u32 (*get_kind_pitch)(void);
1006 u32 (*get_flush_retries)(struct gk20a *g,
1007 enum nvgpu_flush_op op);
1008 } mm;
1009 /*
1010 * This function is called to allocate secure memory (memory
1011 * that the CPU cannot see). The function should fill the
1012 * context buffer descriptor (especially fields destroy, sgt,
1013 * size).
1014 */
1015 int (*secure_alloc)(struct gk20a *g,
1016 struct gr_ctx_buffer_desc *desc,
1017 size_t size);
1018 struct {
1019 void (*exit)(struct gk20a *g, struct nvgpu_mem *mem,
1020 struct nvgpu_sgl *sgl);
1021 u32 (*data032_r)(u32 i);
1022 } pramin;
1023 struct {
1024 int (*init_therm_setup_hw)(struct gk20a *g);
1025 void (*init_elcg_mode)(struct gk20a *g, u32 mode, u32 engine);
1026 void (*init_blcg_mode)(struct gk20a *g, u32 mode, u32 engine);
1027 int (*elcg_init_idle_filters)(struct gk20a *g);
1028#ifdef CONFIG_DEBUG_FS
1029 void (*therm_debugfs_init)(struct gk20a *g);
1030#endif
1031 int (*get_internal_sensor_curr_temp)(struct gk20a *g, u32 *temp_f24_8);
1032 void (*get_internal_sensor_limits)(s32 *max_24_8,
1033 s32 *min_24_8);
1034 u32 (*configure_therm_alert)(struct gk20a *g, s32 curr_warn_temp);
1035 } therm;
1036 struct {
1037 bool (*is_pmu_supported)(struct gk20a *g);
1038 int (*prepare_ucode)(struct gk20a *g);
1039 int (*pmu_setup_hw_and_bootstrap)(struct gk20a *g);
1040 int (*pmu_nsbootstrap)(struct nvgpu_pmu *pmu);
1041 int (*pmu_init_perfmon)(struct nvgpu_pmu *pmu);
1042 int (*pmu_perfmon_start_sampling)(struct nvgpu_pmu *pmu);
1043 int (*pmu_perfmon_stop_sampling)(struct nvgpu_pmu *pmu);
1044 int (*pmu_perfmon_get_samples_rpc)(struct nvgpu_pmu *pmu);
1045 int (*pmu_setup_elpg)(struct gk20a *g);
1046 u32 (*pmu_get_queue_head)(u32 i);
1047 u32 (*pmu_get_queue_head_size)(void);
1048 u32 (*pmu_get_queue_tail_size)(void);
1049 u32 (*pmu_get_queue_tail)(u32 i);
1050 int (*pmu_queue_head)(struct gk20a *g,
1051 struct nvgpu_falcon_queue *queue, u32 *head, bool set);
1052 int (*pmu_queue_tail)(struct gk20a *g,
1053 struct nvgpu_falcon_queue *queue, u32 *tail, bool set);
1054 void (*pmu_msgq_tail)(struct nvgpu_pmu *pmu,
1055 u32 *tail, bool set);
1056 u32 (*pmu_mutex_size)(void);
1057 int (*pmu_mutex_acquire)(struct nvgpu_pmu *pmu,
1058 u32 id, u32 *token);
1059 int (*pmu_mutex_release)(struct nvgpu_pmu *pmu,
1060 u32 id, u32 *token);
1061 bool (*pmu_is_interrupted)(struct nvgpu_pmu *pmu);
1062 void (*pmu_isr)(struct gk20a *g);
1063 void (*pmu_init_perfmon_counter)(struct gk20a *g);
1064 void (*pmu_pg_idle_counter_config)(struct gk20a *g, u32 pg_engine_id);
1065 u32 (*pmu_read_idle_counter)(struct gk20a *g, u32 counter_id);
1066 u32 (*pmu_read_idle_intr_status)(struct gk20a *g);
1067 void (*pmu_clear_idle_intr_status)(struct gk20a *g);
1068 void (*pmu_reset_idle_counter)(struct gk20a *g, u32 counter_id);
1069 void (*pmu_dump_elpg_stats)(struct nvgpu_pmu *pmu);
1070 void (*pmu_dump_falcon_stats)(struct nvgpu_pmu *pmu);
1071 void (*pmu_enable_irq)(struct nvgpu_pmu *pmu, bool enable);
1072 int (*init_wpr_region)(struct gk20a *g);
1073 int (*load_lsfalcon_ucode)(struct gk20a *g, u32 falconidmask);
1074 void (*write_dmatrfbase)(struct gk20a *g, u32 addr);
1075 void (*pmu_elpg_statistics)(struct gk20a *g, u32 pg_engine_id,
1076 struct pmu_pg_stats_data *pg_stat_data);
1077 int (*pmu_pg_init_param)(struct gk20a *g, u32 pg_engine_id);
1078 int (*pmu_pg_set_sub_feature_mask)(struct gk20a *g,
1079 u32 pg_engine_id);
1080 u32 (*pmu_pg_supported_engines_list)(struct gk20a *g);
1081 u32 (*pmu_pg_engines_feature_list)(struct gk20a *g,
1082 u32 pg_engine_id);
1083 int (*pmu_process_pg_event)(struct gk20a *g, void *pmumsg);
1084 bool (*pmu_is_lpwr_feature_supported)(struct gk20a *g,
1085 u32 feature_id);
1086 int (*pmu_lpwr_enable_pg)(struct gk20a *g, bool pstate_lock);
1087 int (*pmu_lpwr_disable_pg)(struct gk20a *g, bool pstate_lock);
1088 u32 (*pmu_pg_param_post_init)(struct gk20a *g);
1089 void (*dump_secure_fuses)(struct gk20a *g);
1090 int (*reset_engine)(struct gk20a *g, bool do_reset);
1091 bool (*is_engine_in_reset)(struct gk20a *g);
1092 bool (*is_lazy_bootstrap)(u32 falcon_id);
1093 bool (*is_priv_load)(u32 falcon_id);
1094 int (*pmu_populate_loader_cfg)(struct gk20a *g,
1095 void *lsfm, u32 *p_bl_gen_desc_size);
1096 int (*flcn_populate_bl_dmem_desc)(struct gk20a *g,
1097 void *lsfm, u32 *p_bl_gen_desc_size, u32 falconid);
1098 void (*handle_ext_irq)(struct gk20a *g, u32 intr);
1099 void (*set_irqmask)(struct gk20a *g);
1100 void (*update_lspmu_cmdline_args)(struct gk20a *g);
1101 void (*setup_apertures)(struct gk20a *g);
1102 u32 (*get_irqdest)(struct gk20a *g);
1103 int (*alloc_super_surface)(struct gk20a *g,
1104 struct nvgpu_mem *super_surface, u32 size);
1105 bool (*is_debug_mode_enabled)(struct gk20a *g);
1106 void (*secured_pmu_start)(struct gk20a *g);
1107 } pmu;
1108 struct {
1109 int (*init_debugfs)(struct gk20a *g);
1110 void (*disable_slowboot)(struct gk20a *g);
1111 int (*init_clk_support)(struct gk20a *g);
1112 int (*suspend_clk_support)(struct gk20a *g);
1113 u32 (*get_crystal_clk_hz)(struct gk20a *g);
1114 int (*clk_domain_get_f_points)(struct gk20a *g,
1115 u32 clkapidomain, u32 *pfpointscount,
1116 u16 *pfreqpointsinmhz);
1117 int (*clk_get_round_rate)(struct gk20a *g, u32 api_domain,
1118 unsigned long rate_target, unsigned long *rounded_rate);
1119 int (*get_clk_range)(struct gk20a *g, u32 api_domain,
1120 u16 *min_mhz, u16 *max_mhz);
1121 unsigned long (*measure_freq)(struct gk20a *g, u32 api_domain);
1122 u32 (*get_rate_cntr)(struct gk20a *g, struct namemap_cfg *c);
1123 unsigned long (*get_rate)(struct gk20a *g, u32 api_domain);
1124 int (*set_rate)(struct gk20a *g, u32 api_domain, unsigned long rate);
1125 unsigned long (*get_fmax_at_vmin_safe)(struct gk20a *g);
1126 u32 (*get_ref_clock_rate)(struct gk20a *g);
1127 int (*predict_mv_at_hz_cur_tfloor)(struct clk_gk20a *clk,
1128 unsigned long rate);
1129 unsigned long (*get_maxrate)(struct gk20a *g, u32 api_domain);
1130 int (*prepare_enable)(struct clk_gk20a *clk);
1131 void (*disable_unprepare)(struct clk_gk20a *clk);
1132 int (*get_voltage)(struct clk_gk20a *clk, u64 *val);
1133 int (*get_gpcclk_clock_counter)(struct clk_gk20a *clk, u64 *val);
1134 int (*pll_reg_write)(struct gk20a *g, u32 reg, u32 val);
1135 int (*get_pll_debug_data)(struct gk20a *g,
1136 struct nvgpu_clk_pll_debug_data *d);
1137 int (*mclk_init)(struct gk20a *g);
1138 void (*mclk_deinit)(struct gk20a *g);
1139 int (*mclk_change)(struct gk20a *g, u16 val);
1140 bool split_rail_support;
1141 bool support_clk_freq_controller;
1142 bool support_pmgr_domain;
1143 bool support_lpwr_pg;
1144 u32 (*perf_pmu_vfe_load)(struct gk20a *g);
1145 u32 lut_num_entries;
1146 } clk;
1147 struct {
1148 int (*arbiter_clk_init)(struct gk20a *g);
1149 u32 (*get_arbiter_clk_domains)(struct gk20a *g);
1150 int (*get_arbiter_f_points)(struct gk20a *g,u32 api_domain,
1151 u32 *num_points, u16 *freqs_in_mhz);
1152 int (*get_arbiter_clk_range)(struct gk20a *g, u32 api_domain,
1153 u16 *min_mhz, u16 *max_mhz);
1154 int (*get_arbiter_clk_default)(struct gk20a *g, u32 api_domain,
1155 u16 *default_mhz);
1156 void (*clk_arb_run_arbiter_cb)(struct nvgpu_clk_arb *arb);
1157 /* This function is inherently unsafe to call while
1158 * arbiter is running arbiter must be blocked
1159 * before calling this function */
1160 int (*get_current_pstate)(struct gk20a *g);
1161 void (*clk_arb_cleanup)(struct nvgpu_clk_arb *arb);
1162 } clk_arb;
1163 struct {
1164 int (*handle_pmu_perf_event)(struct gk20a *g, void *pmu_msg);
1165 } pmu_perf;
1166 struct {
1167 int (*exec_regops)(struct dbg_session_gk20a *dbg_s,
1168 struct nvgpu_dbg_reg_op *ops,
1169 u64 num_ops,
1170 bool *is_current_ctx);
1171 const struct regop_offset_range* (
1172 *get_global_whitelist_ranges)(void);
1173 u64 (*get_global_whitelist_ranges_count)(void);
1174 const struct regop_offset_range* (
1175 *get_context_whitelist_ranges)(void);
1176 u64 (*get_context_whitelist_ranges_count)(void);
1177 const u32* (*get_runcontrol_whitelist)(void);
1178 u64 (*get_runcontrol_whitelist_count)(void);
1179 const u32* (*get_qctl_whitelist)(void);
1180 u64 (*get_qctl_whitelist_count)(void);
1181 } regops;
1182 struct {
1183 void (*intr_mask)(struct gk20a *g);
1184 void (*intr_enable)(struct gk20a *g);
1185 void (*intr_unit_config)(struct gk20a *g,
1186 bool enable, bool is_stalling, u32 mask);
1187 void (*isr_stall)(struct gk20a *g);
1188 bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr);
1189 bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr);
1190 bool (*is_stall_and_eng_intr_pending)(struct gk20a *g,
1191 u32 act_eng_id, u32 *eng_intr_pending);
1192 u32 (*intr_stall)(struct gk20a *g);
1193 void (*intr_stall_pause)(struct gk20a *g);
1194 void (*intr_stall_resume)(struct gk20a *g);
1195 u32 (*intr_nonstall)(struct gk20a *g);
1196 void (*intr_nonstall_pause)(struct gk20a *g);
1197 void (*intr_nonstall_resume)(struct gk20a *g);
1198 u32 (*isr_nonstall)(struct gk20a *g);
1199 void (*enable)(struct gk20a *g, u32 units);
1200 void (*disable)(struct gk20a *g, u32 units);
1201 void (*reset)(struct gk20a *g, u32 units);
1202 bool (*is_enabled)(struct gk20a *g, enum nvgpu_unit unit);
1203 bool (*is_intr1_pending)(struct gk20a *g, enum nvgpu_unit unit, u32 mc_intr_1);
1204 void (*log_pending_intrs)(struct gk20a *g);
1205 void (*fbpa_isr)(struct gk20a *g);
1206 u32 (*reset_mask)(struct gk20a *g, enum nvgpu_unit unit);
1207 void (*fb_reset)(struct gk20a *g);
1208 } mc;
1209 struct {
1210 void (*show_dump)(struct gk20a *g,
1211 struct gk20a_debug_output *o);
1212 } debug;
1213 struct {
1214 int (*dbg_set_powergate)(struct dbg_session_gk20a *dbg_s,
1215 bool disable_powergate);
1216 bool (*check_and_set_global_reservation)(
1217 struct dbg_session_gk20a *dbg_s,
1218 struct dbg_profiler_object_data *prof_obj);
1219 bool (*check_and_set_context_reservation)(
1220 struct dbg_session_gk20a *dbg_s,
1221 struct dbg_profiler_object_data *prof_obj);
1222 void (*release_profiler_reservation)(
1223 struct dbg_session_gk20a *dbg_s,
1224 struct dbg_profiler_object_data *prof_obj);
1225 int (*perfbuffer_enable)(struct gk20a *g, u64 offset, u32 size);
1226 int (*perfbuffer_disable)(struct gk20a *g);
1227 } dbg_session_ops;
1228
1229 u32 (*get_litter_value)(struct gk20a *g, int value);
1230 int (*chip_init_gpu_characteristics)(struct gk20a *g);
1231
1232 struct {
1233 void (*init_hw)(struct gk20a *g);
1234 void (*isr)(struct gk20a *g);
1235 int (*bar1_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst);
1236 int (*bar2_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst);
1237 u32 (*set_bar0_window)(struct gk20a *g, struct nvgpu_mem *mem,
1238 struct nvgpu_sgt *sgt, struct nvgpu_sgl *sgl,
1239 u32 w);
1240 u32 (*read_sw_scratch)(struct gk20a *g, u32 index);
1241 void (*write_sw_scratch)(struct gk20a *g, u32 index, u32 val);
1242 } bus;
1243
1244 struct {
1245 void (*isr)(struct gk20a *g);
1246 int (*read_ptimer)(struct gk20a *g, u64 *value);
1247 int (*get_timestamps_zipper)(struct gk20a *g,
1248 u32 source_id, u32 count,
1249 struct nvgpu_cpu_time_correlation_sample *);
1250 } ptimer;
1251
1252 struct {
1253 int (*init)(struct gk20a *g);
1254 int (*preos_wait_for_halt)(struct gk20a *g);
1255 void (*preos_reload_check)(struct gk20a *g);
1256 int (*devinit)(struct gk20a *g);
1257 int (*preos)(struct gk20a *g);
1258 int (*verify_devinit)(struct gk20a *g);
1259 } bios;
1260
1261#if defined(CONFIG_GK20A_CYCLE_STATS)
1262 struct {
1263 int (*enable_snapshot)(struct channel_gk20a *ch,
1264 struct gk20a_cs_snapshot_client *client);
1265 void (*disable_snapshot)(struct gr_gk20a *gr);
1266 int (*check_data_available)(struct channel_gk20a *ch,
1267 u32 *pending,
1268 bool *hw_overflow);
1269 void (*set_handled_snapshots)(struct gk20a *g, u32 num);
1270 u32 (*allocate_perfmon_ids)(struct gk20a_cs_snapshot *data,
1271 u32 count);
1272 u32 (*release_perfmon_ids)(struct gk20a_cs_snapshot *data,
1273 u32 start,
1274 u32 count);
1275 int (*detach_snapshot)(struct channel_gk20a *ch,
1276 struct gk20a_cs_snapshot_client *client);
1277 bool (*get_overflow_status)(struct gk20a *g);
1278 u32 (*get_pending_snapshots)(struct gk20a *g);
1279 } css;
1280#endif
1281 struct {
1282 int (*get_speed)(struct gk20a *g, u32 *xve_link_speed);
1283 int (*set_speed)(struct gk20a *g, u32 xve_link_speed);
1284 void (*available_speeds)(struct gk20a *g, u32 *speed_mask);
1285 u32 (*xve_readl)(struct gk20a *g, u32 reg);
1286 void (*xve_writel)(struct gk20a *g, u32 reg, u32 val);
1287 void (*disable_aspm)(struct gk20a *g);
1288 void (*reset_gpu)(struct gk20a *g);
1289#if defined(CONFIG_PCI_MSI)
1290 void (*rearm_msi)(struct gk20a *g);
1291#endif
1292 void (*enable_shadow_rom)(struct gk20a *g);
1293 void (*disable_shadow_rom)(struct gk20a *g);
1294 u32 (*get_link_control_status)(struct gk20a *g);
1295 } xve;
1296 struct {
1297 int (*falcon_hal_sw_init)(struct nvgpu_falcon *flcn);
1298 } falcon;
1299 struct {
1300 void (*enable_priv_ring)(struct gk20a *g);
1301 void (*isr)(struct gk20a *g);
1302 void (*decode_error_code)(struct gk20a *g, u32 error_code);
1303 void (*set_ppriv_timeout_settings)(struct gk20a *g);
1304 u32 (*enum_ltc)(struct gk20a *g);
1305 } priv_ring;
1306 struct {
1307 int (*check_priv_security)(struct gk20a *g);
1308 bool (*is_opt_ecc_enable)(struct gk20a *g);
1309 bool (*is_opt_feature_override_disable)(struct gk20a *g);
1310 u32 (*fuse_status_opt_fbio)(struct gk20a *g);
1311 u32 (*fuse_status_opt_fbp)(struct gk20a *g);
1312 u32 (*fuse_status_opt_rop_l2_fbp)(struct gk20a *g, u32 fbp);
1313 u32 (*fuse_status_opt_gpc)(struct gk20a *g);
1314 u32 (*fuse_status_opt_tpc_gpc)(struct gk20a *g, u32 gpc);
1315 void (*fuse_ctrl_opt_tpc_gpc)(struct gk20a *g, u32 gpc, u32 val);
1316 u32 (*fuse_opt_sec_debug_en)(struct gk20a *g);
1317 u32 (*fuse_opt_priv_sec_en)(struct gk20a *g);
1318 u32 (*read_vin_cal_fuse_rev)(struct gk20a *g);
1319 u32 (*read_vin_cal_slope_intercept_fuse)(struct gk20a *g,
1320 u32 vin_id, u32 *slope,
1321 u32 *intercept);
1322 u32 (*read_vin_cal_gain_offset_fuse)(struct gk20a *g,
1323 u32 vin_id, s8 *gain,
1324 s8 *offset);
1325 } fuse;
1326 struct {
1327 int (*init)(struct gk20a *g);
1328 int (*discover_ioctrl)(struct gk20a *g);
1329 int (*discover_link)(struct gk20a *g);
1330 int (*isr)(struct gk20a *g);
1331 int (*rxdet)(struct gk20a *g, u32 link_id);
1332 int (*setup_pll)(struct gk20a *g, unsigned long link_mask);
1333 int (*minion_data_ready_en)(struct gk20a *g,
1334 unsigned long link_mask, bool sync);
1335 void (*get_connected_link_mask)(u32 *link_mask);
1336 void (*set_sw_war)(struct gk20a *g, u32 link_id);
1337 /* API */
1338 int (*link_early_init)(struct gk20a *g, unsigned long mask);
1339 u32 (*link_get_mode)(struct gk20a *g, u32 link_id);
1340 u32 (*link_get_state)(struct gk20a *g, u32 link_id);
1341 int (*link_set_mode)(struct gk20a *g, u32 link_id, u32 mode);
1342 u32 (*get_sublink_mode)(struct gk20a *g, u32 link_id,
1343 bool is_rx_sublink);
1344 u32 (*get_rx_sublink_state)(struct gk20a *g, u32 link_id);
1345 u32 (*get_tx_sublink_state)(struct gk20a *g, u32 link_id);
1346 int (*set_sublink_mode)(struct gk20a *g, u32 link_id,
1347 bool is_rx_sublink, u32 mode);
1348 int (*interface_init)(struct gk20a *g);
1349 int (*interface_disable)(struct gk20a *g);
1350 int (*reg_init)(struct gk20a *g);
1351 int (*shutdown)(struct gk20a *g);
1352 int (*early_init)(struct gk20a *g);
1353 } nvlink;
1354 struct {
1355 u32 (*get_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g);
1356 void (*set_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g, u32 val);
1357 u32 (*get_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g);
1358 void (*set_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g, u32 val);
1359 } top;
1360 struct {
1361 void (*acr_sw_init)(struct gk20a *g, struct nvgpu_acr *acr);
1362 } acr;
1363 struct {
1364 int (*tpc_powergate)(struct gk20a *g, u32 fuse_status);
1365 } tpc;
1366 void (*semaphore_wakeup)(struct gk20a *g, bool post_events);
1367};
1368
1369struct nvgpu_bios_ucode {
1370 u8 *bootloader;
1371 u32 bootloader_phys_base;
1372 u32 bootloader_size;
1373 u8 *ucode;
1374 u32 phys_base;
1375 u32 size;
1376 u8 *dmem;
1377 u32 dmem_phys_base;
1378 u32 dmem_size;
1379 u32 code_entry_point;
1380};
1381
1382struct nvgpu_bios {
1383 u32 vbios_version;
1384 u8 vbios_oem_version;
1385
1386 u8 *data;
1387 size_t size;
1388
1389 struct nvgpu_bios_ucode devinit;
1390 struct nvgpu_bios_ucode preos;
1391
1392 u8 *devinit_tables;
1393 u32 devinit_tables_size;
1394 u8 *bootscripts;
1395 u32 bootscripts_size;
1396
1397 u8 mem_strap_data_count;
1398 u16 mem_strap_xlat_tbl_ptr;
1399
1400 u32 condition_table_ptr;
1401
1402 u32 devinit_tables_phys_base;
1403 u32 devinit_script_phys_base;
1404
1405 struct bit_token *perf_token;
1406 struct bit_token *clock_token;
1407 struct bit_token *virt_token;
1408 u32 expansion_rom_offset;
1409
1410 u32 nvlink_config_data_offset;
1411};
1412
1413struct nvgpu_gpu_params {
1414 /* GPU architecture ID */
1415 u32 gpu_arch;
1416 /* GPU implementation ID */
1417 u32 gpu_impl;
1418 /* GPU revision ID */
1419 u32 gpu_rev;
1420 /* sm version */
1421 u32 sm_arch_sm_version;
1422 /* sm instruction set */
1423 u32 sm_arch_spa_version;
1424 u32 sm_arch_warp_count;
1425};
1426
1427struct gk20a {
1428 void (*free)(struct gk20a *g);
1429 struct nvgpu_nvhost_dev *nvhost_dev;
1430
1431 /*
1432 * Used by <nvgpu/enabled.h>. Do not access directly!
1433 */
1434 unsigned long *enabled_flags;
1435
1436#ifdef __KERNEL__
1437 struct notifier_block nvgpu_reboot_nb;
1438#endif
1439
1440 nvgpu_atomic_t usage_count;
1441
1442 struct nvgpu_mutex ctxsw_disable_lock;
1443 int ctxsw_disable_count;
1444
1445 struct nvgpu_ref refcount;
1446
1447 const char *name;
1448
1449 bool gpu_reset_done;
1450 bool power_on;
1451 bool suspended;
1452 bool sw_ready;
1453
1454 u64 log_mask;
1455 u32 log_trace;
1456
1457 struct nvgpu_mutex tpc_pg_lock;
1458
1459 struct nvgpu_gpu_params params;
1460
1461 /*
1462 * Guards access to hardware when usual gk20a_{busy,idle} are skipped
1463 * for submits and held for channel lifetime but dropped for an ongoing
1464 * gk20a_do_idle().
1465 */
1466 struct nvgpu_rwsem deterministic_busy;
1467
1468 struct nvgpu_falcon pmu_flcn;
1469 struct nvgpu_falcon sec2_flcn;
1470 struct nvgpu_falcon fecs_flcn;
1471 struct nvgpu_falcon gpccs_flcn;
1472 struct nvgpu_falcon nvdec_flcn;
1473 struct nvgpu_falcon minion_flcn;
1474 struct nvgpu_falcon gsp_flcn;
1475 struct clk_gk20a clk;
1476 struct fifo_gk20a fifo;
1477 struct nvgpu_nvlink_dev nvlink;
1478 struct gr_gk20a gr;
1479 struct sim_nvgpu *sim;
1480 struct mm_gk20a mm;
1481 struct nvgpu_pmu pmu;
1482 struct nvgpu_acr acr;
1483 struct nvgpu_ecc ecc;
1484 struct clk_pmupstate clk_pmu;
1485 struct perf_pmupstate perf_pmu;
1486 struct pmgr_pmupstate pmgr_pmu;
1487 struct therm_pmupstate therm_pmu;
1488 struct nvgpu_sec2 sec2;
1489 struct nvgpu_sched_ctrl sched_ctrl;
1490
1491#ifdef CONFIG_DEBUG_FS
1492 struct railgate_stats pstats;
1493#endif
1494 u32 gr_idle_timeout_default;
1495 bool timeouts_disabled_by_user;
1496 unsigned int ch_wdt_timeout_ms;
1497 u32 fifo_eng_timeout_us;
1498
1499 struct nvgpu_mutex power_lock;
1500
1501 /* Channel priorities */
1502 u32 timeslice_low_priority_us;
1503 u32 timeslice_medium_priority_us;
1504 u32 timeslice_high_priority_us;
1505 u32 min_timeslice_us;
1506 u32 max_timeslice_us;
1507 bool runlist_interleave;
1508
1509 struct nvgpu_mutex cg_pg_lock;
1510 bool slcg_enabled;
1511 bool blcg_enabled;
1512 bool elcg_enabled;
1513 bool elpg_enabled;
1514 bool aelpg_enabled;
1515 bool can_elpg;
1516 bool mscg_enabled;
1517 bool forced_idle;
1518 bool forced_reset;
1519 bool allow_all;
1520
1521 u32 ptimer_src_freq;
1522
1523 int railgate_delay;
1524 u8 ldiv_slowdown_factor;
1525 unsigned int aggressive_sync_destroy_thresh;
1526 bool aggressive_sync_destroy;
1527
1528 /* Debugfs knob for forcing syncpt support off in runtime. */
1529 u32 disable_syncpoints;
1530
1531 bool support_pmu;
1532
1533 bool is_virtual;
1534
1535 bool has_cde;
1536
1537 u32 emc3d_ratio;
1538
1539 struct nvgpu_spinlock ltc_enabled_lock;
1540
1541 struct gk20a_ctxsw_ucode_info ctxsw_ucode_info;
1542
1543 /*
1544 * A group of semaphore pools. One for each channel.
1545 */
1546 struct nvgpu_semaphore_sea *sema_sea;
1547
1548 /* held while manipulating # of debug/profiler sessions present */
1549 /* also prevents debug sessions from attaching until released */
1550 struct nvgpu_mutex dbg_sessions_lock;
1551 int dbg_powergating_disabled_refcount; /*refcount for pg disable */
1552 /*refcount for timeout disable */
1553 nvgpu_atomic_t timeouts_disabled_refcount;
1554
1555 /* must have dbg_sessions_lock before use */
1556 struct nvgpu_dbg_reg_op *dbg_regops_tmp_buf;
1557 u32 dbg_regops_tmp_buf_ops;
1558
1559 /* For perfbuf mapping */
1560 struct {
1561 struct dbg_session_gk20a *owner;
1562 u64 offset;
1563 } perfbuf;
1564
1565 /* For profiler reservations */
1566 struct nvgpu_list_node profiler_objects;
1567 bool global_profiler_reservation_held;
1568 int profiler_reservation_count;
1569
1570 void (*remove_support)(struct gk20a *);
1571
1572 u64 pg_ingating_time_us;
1573 u64 pg_ungating_time_us;
1574 u32 pg_gating_cnt;
1575
1576 struct nvgpu_spinlock mc_enable_lock;
1577
1578 struct gk20a_as as;
1579
1580 struct nvgpu_mutex client_lock;
1581 int client_refcount; /* open channels and ctrl nodes */
1582
1583 struct gpu_ops ops;
1584 u32 mc_intr_mask_restore[4];
1585 /*used for change of enum zbc update cmd id from ver 0 to ver1*/
1586 u32 pmu_ver_cmd_id_zbc_table_update;
1587 u32 pmu_lsf_pmu_wpr_init_done;
1588 u32 pmu_lsf_loaded_falcon_id;
1589
1590 int irqs_enabled;
1591 int irq_stall; /* can be same as irq_nonstall in case of PCI */
1592 int irq_nonstall;
1593 u32 max_ltc_count;
1594 u32 ltc_count;
1595 u32 ltc_streamid;
1596
1597 struct gk20a_worker {
1598 struct nvgpu_thread poll_task;
1599 nvgpu_atomic_t put;
1600 struct nvgpu_cond wq;
1601 struct nvgpu_list_node items;
1602 struct nvgpu_spinlock items_lock;
1603 struct nvgpu_mutex start_lock;
1604 } channel_worker, clk_arb_worker;
1605
1606 struct {
1607 void (*open)(struct channel_gk20a *ch);
1608 void (*close)(struct channel_gk20a *ch);
1609 void (*work_completion_signal)(struct channel_gk20a *ch);
1610 void (*work_completion_cancel_sync)(struct channel_gk20a *ch);
1611 bool (*os_fence_framework_inst_exists)(struct channel_gk20a *ch);
1612 int (*init_os_fence_framework)(
1613 struct channel_gk20a *ch, const char *fmt, ...);
1614 void (*signal_os_fence_framework)(struct channel_gk20a *ch);
1615 void (*destroy_os_fence_framework)(struct channel_gk20a *ch);
1616 int (*copy_user_gpfifo)(struct nvgpu_gpfifo_entry *dest,
1617 struct nvgpu_gpfifo_userdata userdata,
1618 u32 start, u32 length);
1619 int (*alloc_usermode_buffers)(struct channel_gk20a *c,
1620 struct nvgpu_setup_bind_args *args);
1621 void (*free_usermode_buffers)(struct channel_gk20a *c);
1622 } os_channel;
1623
1624 struct gk20a_scale_profile *scale_profile;
1625 unsigned long last_freq;
1626
1627 struct gk20a_ctxsw_trace *ctxsw_trace;
1628 struct gk20a_fecs_trace *fecs_trace;
1629
1630 bool mmu_debug_ctrl;
1631 u32 mmu_debug_mode_refcnt;
1632
1633 u32 tpc_fs_mask_user;
1634
1635 u32 tpc_pg_mask;
1636 u32 tpc_count;
1637 bool can_tpc_powergate;
1638
1639 u32 valid_tpc_mask[MAX_TPC_PG_CONFIGS];
1640
1641 struct nvgpu_bios bios;
1642 bool bios_is_init;
1643
1644 struct nvgpu_clk_arb *clk_arb;
1645
1646 struct nvgpu_mutex clk_arb_enable_lock;
1647
1648 nvgpu_atomic_t clk_arb_global_nr;
1649
1650 struct gk20a_ce_app ce_app;
1651
1652 bool ltc_intr_en_illegal_compstat;
1653
1654 /* PCI device identifier */
1655 u16 pci_vendor_id, pci_device_id;
1656 u16 pci_subsystem_vendor_id, pci_subsystem_device_id;
1657 u16 pci_class;
1658 u8 pci_revision;
1659
1660 /*
1661 * PCI power management: i2c device index, port and address for
1662 * INA3221.
1663 */
1664 u32 ina3221_dcb_index;
1665 u32 ina3221_i2c_address;
1666 u32 ina3221_i2c_port;
1667 bool hardcode_sw_threshold;
1668
1669 /* PCIe power states. */
1670 bool xve_l0s;
1671 bool xve_l1;
1672
1673 /* Current warning temp in sfxp24.8 */
1674 s32 curr_warn_temp;
1675
1676#if defined(CONFIG_PCI_MSI)
1677 /* Check if msi is enabled */
1678 bool msi_enabled;
1679#endif
1680#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
1681 struct nvgpu_mem_alloc_tracker *vmallocs;
1682 struct nvgpu_mem_alloc_tracker *kmallocs;
1683#endif
1684
1685 /* The minimum VBIOS version supported */
1686 u32 vbios_min_version;
1687
1688 /* memory training sequence and mclk switch scripts */
1689 u32 mem_config_idx;
1690
1691 u64 dma_memory_used;
1692
1693#if defined(CONFIG_TEGRA_GK20A_NVHOST)
1694 u64 syncpt_unit_base;
1695 size_t syncpt_unit_size;
1696 u32 syncpt_size;
1697#endif
1698 struct nvgpu_mem syncpt_mem;
1699
1700 struct nvgpu_list_node boardobj_head;
1701 struct nvgpu_list_node boardobjgrp_head;
1702
1703 struct nvgpu_mem pdb_cache_war_mem;
1704};
1705
1706static inline bool nvgpu_is_timeouts_enabled(struct gk20a *g)
1707{
1708 return nvgpu_atomic_read(&g->timeouts_disabled_refcount) == 0;
1709}
1710
1711static inline u32 gk20a_get_gr_idle_timeout(struct gk20a *g)
1712{
1713 return nvgpu_is_timeouts_enabled(g) ?
1714 g->gr_idle_timeout_default : UINT_MAX;
1715}
1716
1717#define MULTICHAR_TAG(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d))
1718enum BAR0_DEBUG_OPERATION {
1719 BARO_ZERO_NOP = 0,
1720 OP_END = MULTICHAR_TAG('D', 'O', 'N', 'E'),
1721 BAR0_READ32 = MULTICHAR_TAG('0', 'R', '3', '2'),
1722 BAR0_WRITE32 = MULTICHAR_TAG('0', 'W', '3', '2'),
1723};
1724
1725struct share_buffer_head {
1726 enum BAR0_DEBUG_OPERATION operation;
1727/* size of the operation item */
1728 u32 size;
1729 u32 completed;
1730 u32 failed;
1731 u64 context;
1732 u64 completion_callback;
1733};
1734
1735struct gk20a_cyclestate_buffer_elem {
1736 struct share_buffer_head head;
1737/* in */
1738 u64 p_data;
1739 u64 p_done;
1740 u32 offset_bar0;
1741 u16 first_bit;
1742 u16 last_bit;
1743/* out */
1744/* keep 64 bits to be consistent */
1745 u64 data;
1746};
1747
1748/* operations that will need to be executed on non stall workqueue */
1749#define GK20A_NONSTALL_OPS_WAKEUP_SEMAPHORE BIT32(0)
1750#define GK20A_NONSTALL_OPS_POST_EVENTS BIT32(1)
1751
1752/* register accessors */
1753void __nvgpu_check_gpu_state(struct gk20a *g);
1754void __gk20a_warn_on_no_regs(void);
1755
1756/* classes that the device supports */
1757/* TBD: get these from an open-sourced SDK? */
1758enum {
1759 FERMI_TWOD_A = 0x902D,
1760 KEPLER_INLINE_TO_MEMORY_A = 0xA040,
1761 KEPLER_DMA_COPY_A = 0xA0B5,
1762};
1763
1764#define GK20A_BAR0_IORESOURCE_MEM 0
1765#define GK20A_BAR1_IORESOURCE_MEM 1
1766#define GK20A_SIM_IORESOURCE_MEM 2
1767
1768void gk20a_busy_noresume(struct gk20a *g);
1769void gk20a_idle_nosuspend(struct gk20a *g);
1770int __must_check gk20a_busy(struct gk20a *g);
1771void gk20a_idle(struct gk20a *g);
1772int __gk20a_do_idle(struct gk20a *g, bool force_reset);
1773int __gk20a_do_unidle(struct gk20a *g);
1774
1775int gk20a_wait_for_idle(struct gk20a *g);
1776
1777#define NVGPU_GPU_ARCHITECTURE_SHIFT 4
1778
1779/* constructs unique and compact GPUID from nvgpu_gpu_characteristics
1780 * arch/impl fields */
1781#define GK20A_GPUID(arch, impl) ((u32) ((arch) | (impl)))
1782
1783#define GK20A_GPUID_GK20A 0x000000EA
1784#define GK20A_GPUID_GM20B 0x0000012B
1785#define GK20A_GPUID_GM20B_B 0x0000012E
1786#define NVGPU_GPUID_GP10B 0x0000013B
1787#define NVGPU_GPUID_GP104 0x00000134
1788#define NVGPU_GPUID_GP106 0x00000136
1789#define NVGPU_GPUID_GV11B 0x0000015B
1790#define NVGPU_GPUID_GV100 0x00000140
1791
1792int gk20a_init_gpu_characteristics(struct gk20a *g);
1793
1794bool gk20a_check_poweron(struct gk20a *g);
1795int gk20a_prepare_poweroff(struct gk20a *g);
1796int gk20a_finalize_poweron(struct gk20a *g);
1797
1798int nvgpu_wait_for_stall_interrupts(struct gk20a *g, u32 timeout);
1799int nvgpu_wait_for_nonstall_interrupts(struct gk20a *g, u32 timeout);
1800void nvgpu_wait_for_deferred_interrupts(struct gk20a *g);
1801
1802struct gk20a * __must_check gk20a_get(struct gk20a *g);
1803void gk20a_put(struct gk20a *g);
1804
1805bool nvgpu_has_syncpoints(struct gk20a *g);
1806
1807#endif /* GK20A_H */
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-30 14:22:39 -0400