summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorddutta <ddutta@nvidia.com>2018-09-11 04:56:27 -0400
committermobile promotions <svcmobile_promotions@nvidia.com>2018-09-12 10:16:27 -0400
commit19434a22454778a14cd918ad6a0ec061634a1fc2 (patch)
treea3378f2792882bee8aae949bb60e4530b1f350a3
parent092664656c7f70156bf8b0c444b3e3ea1b533323 (diff)
gpu: nvgpu: move gk20a.h to include/nvgpu/gk20a.h
Move the contents of the gk20a/gk20a.h to <nvgpu/gk20a.h>. In order to enable a smooth transition, include <nvgpu/gk20a.h> in the current file (i.e. gk20a/gk20a.h). Jira NVGPU-597 Change-Id: I998da0e7688a5827e2974e88ab8ad6849083aa4a Signed-off-by: Debarshi Dutta <ddutta@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/1813140 Reviewed-by: Deepak Nibade <dnibade@nvidia.com> Reviewed-by: svc-misra-checker <svc-misra-checker@nvidia.com> GVS: Gerrit_Virtual_Submit Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com> Reviewed-by: Vijayakumar Subbu <vsubbu@nvidia.com> Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Diffstat
-rw-r--r--drivers/gpu/nvgpu/gk20a/gk20a.h1739
-rw-r--r--drivers/gpu/nvgpu/include/nvgpu/gk20a.h1758
2 files changed, 1765 insertions, 1732 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index dd63a105..0df88edf 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -1,4 +1,6 @@
1/* 1/*
2 * This file is used as a temporary redirection header for <nvgpu/gk20a.h>
3 *
2 * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. 4 * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
3 * 5 *
4 * GK20A Graphics 6 * GK20A Graphics
@@ -21,1738 +23,11 @@
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE. 24 * DEALINGS IN THE SOFTWARE.
23 */ 25 */
24#ifndef GK20A_H
25#define GK20A_H
26
27struct gk20a;
28struct fifo_gk20a;
29struct channel_gk20a;
30struct gr_gk20a;
31struct sim_nvgpu;
32struct gk20a_ctxsw_ucode_segments;
33struct gk20a_fecs_trace;
34struct gk20a_ctxsw_trace;
35struct acr_desc;
36struct nvgpu_mem_alloc_tracker;
37struct dbg_profiler_object_data;
38struct gk20a_debug_output;
39struct nvgpu_clk_pll_debug_data;
40struct nvgpu_nvhost_dev;
41struct nvgpu_cpu_time_correlation_sample;
42struct nvgpu_mem_sgt;
43struct nvgpu_warpstate;
44struct nvgpu_clk_session;
45struct nvgpu_clk_arb;
46#ifdef CONFIG_GK20A_CTXSW_TRACE
47struct nvgpu_gpu_ctxsw_trace_filter;
48#endif
49struct priv_cmd_entry;
50struct nvgpu_gpfifo_args;
51
52#include <nvgpu/lock.h>
53#include <nvgpu/thread.h>
54
55#include <nvgpu/mm.h>
56#include <nvgpu/as.h>
57#include <nvgpu/log.h>
58#include <nvgpu/pramin.h>
59#include <nvgpu/acr/nvgpu_acr.h>
60#include <nvgpu/kref.h>
61#include <nvgpu/falcon.h>
62#include <nvgpu/pmu.h>
63#include <nvgpu/atomic.h>
64#include <nvgpu/barrier.h>
65#include <nvgpu/rwsem.h>
66#include <nvgpu/nvlink.h>
67#include <nvgpu/sim.h>
68#include <nvgpu/ecc.h>
69
70#include "clk_gk20a.h"
71#include "ce2_gk20a.h"
72#include "fifo_gk20a.h"
73#include "tsg_gk20a.h"
74#include "clk/clk.h"
75#include "perf/perf.h"
76#include "pmgr/pmgr.h"
77#include "therm/thrm.h"
78
79#ifdef CONFIG_DEBUG_FS
80struct railgate_stats {
81 unsigned long last_rail_gate_start;
82 unsigned long last_rail_gate_complete;
83 unsigned long last_rail_ungate_start;
84 unsigned long last_rail_ungate_complete;
85 unsigned long total_rail_gate_time_ms;
86 unsigned long total_rail_ungate_time_ms;
87 unsigned long railgating_cycle_count;
88};
89#endif
90
91enum gk20a_cbc_op {
92 gk20a_cbc_op_clear,
93 gk20a_cbc_op_clean,
94 gk20a_cbc_op_invalidate,
95};
96
97#define MC_INTR_UNIT_DISABLE false
98#define MC_INTR_UNIT_ENABLE true
99
100#define GPU_LIT_NUM_GPCS 0
101#define GPU_LIT_NUM_PES_PER_GPC 1
102#define GPU_LIT_NUM_ZCULL_BANKS 2
103#define GPU_LIT_NUM_TPC_PER_GPC 3
104#define GPU_LIT_NUM_SM_PER_TPC 4
105#define GPU_LIT_NUM_FBPS 5
106#define GPU_LIT_GPC_BASE 6
107#define GPU_LIT_GPC_STRIDE 7
108#define GPU_LIT_GPC_SHARED_BASE 8
109#define GPU_LIT_TPC_IN_GPC_BASE 9
110#define GPU_LIT_TPC_IN_GPC_STRIDE 10
111#define GPU_LIT_TPC_IN_GPC_SHARED_BASE 11
112#define GPU_LIT_PPC_IN_GPC_BASE 12
113#define GPU_LIT_PPC_IN_GPC_STRIDE 13
114#define GPU_LIT_PPC_IN_GPC_SHARED_BASE 14
115#define GPU_LIT_ROP_BASE 15
116#define GPU_LIT_ROP_STRIDE 16
117#define GPU_LIT_ROP_SHARED_BASE 17
118#define GPU_LIT_HOST_NUM_ENGINES 18
119#define GPU_LIT_HOST_NUM_PBDMA 19
120#define GPU_LIT_LTC_STRIDE 20
121#define GPU_LIT_LTS_STRIDE 21
122#define GPU_LIT_NUM_FBPAS 22
123#define GPU_LIT_FBPA_STRIDE 23
124#define GPU_LIT_FBPA_BASE 24
125#define GPU_LIT_FBPA_SHARED_BASE 25
126#define GPU_LIT_SM_PRI_STRIDE 26
127#define GPU_LIT_SMPC_PRI_BASE 27
128#define GPU_LIT_SMPC_PRI_SHARED_BASE 28
129#define GPU_LIT_SMPC_PRI_UNIQUE_BASE 29
130#define GPU_LIT_SMPC_PRI_STRIDE 30
131#define GPU_LIT_TWOD_CLASS 31
132#define GPU_LIT_THREED_CLASS 32
133#define GPU_LIT_COMPUTE_CLASS 33
134#define GPU_LIT_GPFIFO_CLASS 34
135#define GPU_LIT_I2M_CLASS 35
136#define GPU_LIT_DMA_COPY_CLASS 36
137#define GPU_LIT_GPC_PRIV_STRIDE 37
138
139#define nvgpu_get_litter_value(g, v) (g)->ops.get_litter_value((g), v)
140
141enum nvgpu_unit;
142
143enum nvgpu_flush_op;
144enum gk20a_mem_rw_flag;
145
146struct _resmgr_context;
147struct nvgpu_gpfifo_entry;
148
149struct nvgpu_gpfifo_userdata {
150 struct nvgpu_gpfifo_entry __user *entries;
151 struct _resmgr_context *context;
152};
153
154/*
155 * gpu_ops should only contain function pointers! Non-function pointer members
156 * should go in struct gk20a or be implemented with the boolean flag API defined
157 * in nvgpu/enabled.h
158 */
159
160/* index for FB fault buffer functions */
161#define NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX 0U
162#define NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX 1U
163#define NVGPU_FB_MMU_FAULT_BUF_DISABLED 0U
164#define NVGPU_FB_MMU_FAULT_BUF_ENABLED 1U
165
166/* Parameters for init_elcg_mode/init_blcg_mode */
167enum {
168 ELCG_RUN, /* clk always run, i.e. disable elcg */
169 ELCG_STOP, /* clk is stopped */
170 ELCG_AUTO /* clk will run when non-idle, standard elcg mode */
171};
172
173enum {
174 BLCG_RUN, /* clk always run, i.e. disable blcg */
175 BLCG_AUTO /* clk will run when non-idle, standard blcg mode */
176};
177
178struct gpu_ops {
179 struct {
180 int (*determine_L2_size_bytes)(struct gk20a *gk20a);
181 u64 (*get_cbc_base_divisor)(struct gk20a *g);
182 int (*init_comptags)(struct gk20a *g, struct gr_gk20a *gr);
183 int (*cbc_ctrl)(struct gk20a *g, enum gk20a_cbc_op op,
184 u32 min, u32 max);
185 void (*set_zbc_color_entry)(struct gk20a *g,
186 struct zbc_entry *color_val,
187 u32 index);
188 void (*set_zbc_depth_entry)(struct gk20a *g,
189 struct zbc_entry *depth_val,
190 u32 index);
191 void (*set_zbc_s_entry)(struct gk20a *g,
192 struct zbc_entry *s_val,
193 u32 index);
194 void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr);
195 void (*set_enabled)(struct gk20a *g, bool enabled);
196 void (*init_fs_state)(struct gk20a *g);
197 void (*isr)(struct gk20a *g);
198 u32 (*cbc_fix_config)(struct gk20a *g, int base);
199 void (*flush)(struct gk20a *g);
200 void (*intr_en_illegal_compstat)(struct gk20a *g, bool enable);
201 bool (*pri_is_ltc_addr)(struct gk20a *g, u32 addr);
202 bool (*is_ltcs_ltss_addr)(struct gk20a *g, u32 addr);
203 bool (*is_ltcn_ltss_addr)(struct gk20a *g, u32 addr);
204 void (*split_lts_broadcast_addr)(struct gk20a *g, u32 addr,
205 u32 *priv_addr_table,
206 u32 *priv_addr_table_index);
207 void (*split_ltc_broadcast_addr)(struct gk20a *g, u32 addr,
208 u32 *priv_addr_table,
209 u32 *priv_addr_table_index);
210 } ltc;
211 struct {
212 void (*isr_stall)(struct gk20a *g, u32 inst_id, u32 pri_base);
213 u32 (*isr_nonstall)(struct gk20a *g, u32 inst_id, u32 pri_base);
214 u32 (*get_num_pce)(struct gk20a *g);
215 } ce2;
216 struct {
217 u32 (*get_patch_slots)(struct gk20a *g);
218 int (*init_fs_state)(struct gk20a *g);
219 int (*init_preemption_state)(struct gk20a *g);
220 void (*access_smpc_reg)(struct gk20a *g, u32 quad, u32 offset);
221 void (*bundle_cb_defaults)(struct gk20a *g);
222 void (*cb_size_default)(struct gk20a *g);
223 int (*calc_global_ctx_buffer_size)(struct gk20a *g);
224 void (*commit_global_attrib_cb)(struct gk20a *g,
225 struct nvgpu_gr_ctx *ch_ctx,
226 u64 addr, bool patch);
227 void (*commit_global_bundle_cb)(struct gk20a *g,
228 struct nvgpu_gr_ctx *ch_ctx,
229 u64 addr, u64 size, bool patch);
230 int (*commit_global_cb_manager)(struct gk20a *g,
231 struct channel_gk20a *ch,
232 bool patch);
233 void (*commit_global_pagepool)(struct gk20a *g,
234 struct nvgpu_gr_ctx *ch_ctx,
235 u64 addr, u32 size, bool patch);
236 void (*init_gpc_mmu)(struct gk20a *g);
237 int (*handle_sw_method)(struct gk20a *g, u32 addr,
238 u32 class_num, u32 offset, u32 data);
239 void (*set_alpha_circular_buffer_size)(struct gk20a *g,
240 u32 data);
241 void (*set_circular_buffer_size)(struct gk20a *g, u32 data);
242 void (*set_bes_crop_debug3)(struct gk20a *g, u32 data);
243 void (*set_bes_crop_debug4)(struct gk20a *g, u32 data);
244 void (*enable_hww_exceptions)(struct gk20a *g);
245 bool (*is_valid_class)(struct gk20a *g, u32 class_num);
246 bool (*is_valid_gfx_class)(struct gk20a *g, u32 class_num);
247 bool (*is_valid_compute_class)(struct gk20a *g, u32 class_num);
248 void (*get_sm_dsm_perf_regs)(struct gk20a *g,
249 u32 *num_sm_dsm_perf_regs,
250 u32 **sm_dsm_perf_regs,
251 u32 *perf_register_stride);
252 void (*get_sm_dsm_perf_ctrl_regs)(struct gk20a *g,
253 u32 *num_sm_dsm_perf_regs,
254 u32 **sm_dsm_perf_regs,
255 u32 *perf_register_stride);
256 void (*get_ovr_perf_regs)(struct gk20a *g,
257 u32 *num_ovr_perf_regs,
258 u32 **ovr_perf_regsr);
259 void (*set_hww_esr_report_mask)(struct gk20a *g);
260 int (*setup_alpha_beta_tables)(struct gk20a *g,
261 struct gr_gk20a *gr);
262 int (*falcon_load_ucode)(struct gk20a *g,
263 u64 addr_base,
264 struct gk20a_ctxsw_ucode_segments *segments,
265 u32 reg_offset);
266 int (*load_ctxsw_ucode)(struct gk20a *g);
267 u32 (*get_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
268 void (*set_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
269 int (*alloc_obj_ctx)(struct channel_gk20a *c,
270 u32 class_num, u32 flags);
271 int (*bind_ctxsw_zcull)(struct gk20a *g, struct gr_gk20a *gr,
272 struct channel_gk20a *c, u64 zcull_va,
273 u32 mode);
274 int (*get_zcull_info)(struct gk20a *g, struct gr_gk20a *gr,
275 struct gr_zcull_info *zcull_params);
276 int (*decode_egpc_addr)(struct gk20a *g,
277 u32 addr, enum ctxsw_addr_type *addr_type,
278 u32 *gpc_num, u32 *tpc_num, u32 *broadcast_flags);
279 void (*egpc_etpc_priv_addr_table)(struct gk20a *g, u32 addr,
280 u32 gpc, u32 tpc, u32 broadcast_flags,
281 u32 *priv_addr_table,
282 u32 *priv_addr_table_index);
283 bool (*is_tpc_addr)(struct gk20a *g, u32 addr);
284 bool (*is_egpc_addr)(struct gk20a *g, u32 addr);
285 bool (*is_etpc_addr)(struct gk20a *g, u32 addr);
286 void (*get_egpc_etpc_num)(struct gk20a *g, u32 addr,
287 u32 *gpc_num, u32 *tpc_num);
288 u32 (*get_tpc_num)(struct gk20a *g, u32 addr);
289 u32 (*get_egpc_base)(struct gk20a *g);
290 void (*detect_sm_arch)(struct gk20a *g);
291 int (*add_zbc_color)(struct gk20a *g, struct gr_gk20a *gr,
292 struct zbc_entry *color_val, u32 index);
293 int (*add_zbc_depth)(struct gk20a *g, struct gr_gk20a *gr,
294 struct zbc_entry *depth_val, u32 index);
295 int (*add_zbc_s)(struct gk20a *g, struct gr_gk20a *gr,
296 struct zbc_entry *s_val, u32 index);
297 int (*zbc_set_table)(struct gk20a *g, struct gr_gk20a *gr,
298 struct zbc_entry *zbc_val);
299 int (*zbc_query_table)(struct gk20a *g, struct gr_gk20a *gr,
300 struct zbc_query_params *query_params);
301 int (*zbc_s_query_table)(struct gk20a *g, struct gr_gk20a *gr,
302 struct zbc_query_params *query_params);
303 int (*load_zbc_s_default_tbl)(struct gk20a *g,
304 struct gr_gk20a *gr);
305 int (*load_zbc_s_tbl)(struct gk20a *g,
306 struct gr_gk20a *gr);
307 void (*pmu_save_zbc)(struct gk20a *g, u32 entries);
308 int (*add_zbc)(struct gk20a *g, struct gr_gk20a *gr,
309 struct zbc_entry *zbc_val);
310 bool (*add_zbc_type_s)(struct gk20a *g, struct gr_gk20a *gr,
311 struct zbc_entry *zbc_val, int *ret_val);
312 u32 (*pagepool_default_size)(struct gk20a *g);
313 int (*init_ctx_state)(struct gk20a *g);
314 int (*alloc_gr_ctx)(struct gk20a *g,
315 struct nvgpu_gr_ctx *gr_ctx, struct vm_gk20a *vm,
316 u32 class, u32 padding);
317 void (*free_gr_ctx)(struct gk20a *g,
318 struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx);
319 void (*powergate_tpc)(struct gk20a *g);
320 void (*update_ctxsw_preemption_mode)(struct gk20a *g,
321 struct channel_gk20a *c,
322 struct nvgpu_mem *mem);
323 int (*update_smpc_ctxsw_mode)(struct gk20a *g,
324 struct channel_gk20a *c,
325 bool enable);
326 u32 (*get_hw_accessor_stream_out_mode)(void);
327 int (*update_hwpm_ctxsw_mode)(struct gk20a *g,
328 struct channel_gk20a *c,
329 u64 gpu_va,
330 u32 mode);
331 void (*init_hwpm_pmm_register)(struct gk20a *g);
332 int (*dump_gr_regs)(struct gk20a *g,
333 struct gk20a_debug_output *o);
334 int (*update_pc_sampling)(struct channel_gk20a *ch,
335 bool enable);
336 u32 (*get_max_fbps_count)(struct gk20a *g);
337 u32 (*get_fbp_en_mask)(struct gk20a *g);
338 u32 (*get_max_ltc_per_fbp)(struct gk20a *g);
339 u32 (*get_max_lts_per_ltc)(struct gk20a *g);
340 u32* (*get_rop_l2_en_mask)(struct gk20a *g);
341 void (*init_sm_dsm_reg_info)(void);
342 void (*init_ovr_sm_dsm_perf)(void);
343 int (*wait_empty)(struct gk20a *g, unsigned long duration_ms,
344 u32 expect_delay);
345 void (*init_cyclestats)(struct gk20a *g);
346 void (*enable_cde_in_fecs)(struct gk20a *g,
347 struct nvgpu_mem *mem);
348 int (*set_sm_debug_mode)(struct gk20a *g, struct channel_gk20a *ch,
349 u64 sms, bool enable);
350 void (*bpt_reg_info)(struct gk20a *g,
351 struct nvgpu_warpstate *w_state);
352 void (*get_access_map)(struct gk20a *g,
353 u32 **whitelist, int *num_entries);
354 int (*handle_fecs_error)(struct gk20a *g,
355 struct channel_gk20a *ch,
356 struct gr_gk20a_isr_data *isr_data);
357 int (*pre_process_sm_exception)(struct gk20a *g,
358 u32 gpc, u32 tpc, u32 sm, u32 global_esr, u32 warp_esr,
359 bool sm_debugger_attached,
360 struct channel_gk20a *fault_ch,
361 bool *early_exit, bool *ignore_debugger);
362 u32 (*get_sm_hww_warp_esr)(struct gk20a *g,
363 u32 gpc, u32 tpc, u32 sm);
364 u32 (*get_sm_hww_global_esr)(struct gk20a *g,
365 u32 gpc, u32 tpc, u32 sm);
366 u32 (*get_sm_no_lock_down_hww_global_esr_mask)(struct gk20a *g);
367 int (*lock_down_sm)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm,
368 u32 global_esr_mask, bool check_errors);
369 int (*wait_for_sm_lock_down)(struct gk20a *g, u32 gpc, u32 tpc,
370 u32 sm, u32 global_esr_mask, bool check_errors);
371 void (*clear_sm_hww)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm,
372 u32 global_esr);
373 void (*get_esr_sm_sel)(struct gk20a *g, u32 gpc, u32 tpc,
374 u32 *esr_sm_sel);
375 int (*handle_tpc_sm_ecc_exception)(struct gk20a *g,
376 u32 gpc, u32 tpc,
377 bool *post_event, struct channel_gk20a *fault_ch,
378 u32 *hww_global_esr);
379 int (*handle_sm_exception)(struct gk20a *g,
380 u32 gpc, u32 tpc, u32 sm,
381 bool *post_event, struct channel_gk20a *fault_ch,
382 u32 *hww_global_esr);
383 int (*handle_gcc_exception)(struct gk20a *g, u32 gpc, u32 tpc,
384 bool *post_event, struct channel_gk20a *fault_ch,
385 u32 *hww_global_esr);
386 int (*handle_tex_exception)(struct gk20a *g, u32 gpc, u32 tpc,
387 bool *post_event);
388 int (*handle_tpc_mpc_exception)(struct gk20a *g,
389 u32 gpc, u32 tpc, bool *post_event);
390 int (*handle_gpc_gpccs_exception)(struct gk20a *g, u32 gpc,
391 u32 gpc_exception);
392 int (*handle_gpc_gpcmmu_exception)(struct gk20a *g, u32 gpc,
393 u32 gpc_exception);
394 void (*enable_gpc_exceptions)(struct gk20a *g);
395 void (*enable_exceptions)(struct gk20a *g);
396 int (*init_ecc)(struct gk20a *g);
397 u32 (*get_lrf_tex_ltc_dram_override)(struct gk20a *g);
398 int (*record_sm_error_state)(struct gk20a *g, u32 gpc, u32 tpc,
399 u32 sm, struct channel_gk20a *fault_ch);
400 int (*update_sm_error_state)(struct gk20a *g,
401 struct channel_gk20a *ch, u32 sm_id,
402 struct nvgpu_tsg_sm_error_state *sm_error_state);
403 int (*clear_sm_error_state)(struct gk20a *g,
404 struct channel_gk20a *ch, u32 sm_id);
405 int (*suspend_contexts)(struct gk20a *g,
406 struct dbg_session_gk20a *dbg_s,
407 int *ctx_resident_ch_fd);
408 int (*resume_contexts)(struct gk20a *g,
409 struct dbg_session_gk20a *dbg_s,
410 int *ctx_resident_ch_fd);
411 int (*set_preemption_mode)(struct channel_gk20a *ch,
412 u32 graphics_preempt_mode,
413 u32 compute_preempt_mode);
414 int (*get_preemption_mode_flags)(struct gk20a *g,
415 struct nvgpu_preemption_modes_rec *preemption_modes_rec);
416 int (*set_ctxsw_preemption_mode)(struct gk20a *g,
417 struct nvgpu_gr_ctx *gr_ctx,
418 struct vm_gk20a *vm, u32 class,
419 u32 graphics_preempt_mode,
420 u32 compute_preempt_mode);
421 int (*set_boosted_ctx)(struct channel_gk20a *ch, bool boost);
422 void (*update_boosted_ctx)(struct gk20a *g,
423 struct nvgpu_mem *mem,
424 struct nvgpu_gr_ctx *gr_ctx);
425 int (*init_sm_id_table)(struct gk20a *g);
426 int (*load_smid_config)(struct gk20a *g);
427 void (*program_sm_id_numbering)(struct gk20a *g,
428 u32 gpc, u32 tpc, u32 smid);
429 void (*program_active_tpc_counts)(struct gk20a *g, u32 gpc);
430 int (*setup_rop_mapping)(struct gk20a *g, struct gr_gk20a *gr);
431 int (*init_sw_veid_bundle)(struct gk20a *g);
432 void (*program_zcull_mapping)(struct gk20a *g,
433 u32 zcull_alloc_num, u32 *zcull_map_tiles);
434 int (*commit_global_timeslice)(struct gk20a *g,
435 struct channel_gk20a *c);
436 int (*commit_inst)(struct channel_gk20a *c, u64 gpu_va);
437 void (*write_zcull_ptr)(struct gk20a *g,
438 struct nvgpu_mem *mem, u64 gpu_va);
439 void (*write_pm_ptr)(struct gk20a *g,
440 struct nvgpu_mem *mem, u64 gpu_va);
441 void (*set_preemption_buffer_va)(struct gk20a *g,
442 struct nvgpu_mem *mem, u64 gpu_va);
443 void (*load_tpc_mask)(struct gk20a *g);
444 int (*trigger_suspend)(struct gk20a *g);
445 int (*wait_for_pause)(struct gk20a *g, struct nvgpu_warpstate *w_state);
446 int (*resume_from_pause)(struct gk20a *g);
447 int (*clear_sm_errors)(struct gk20a *g);
448 u32 (*tpc_enabled_exceptions)(struct gk20a *g);
449 int (*set_czf_bypass)(struct gk20a *g,
450 struct channel_gk20a *ch);
451 void (*init_czf_bypass)(struct gk20a *g);
452 bool (*sm_debugger_attached)(struct gk20a *g);
453 void (*suspend_single_sm)(struct gk20a *g,
454 u32 gpc, u32 tpc, u32 sm,
455 u32 global_esr_mask, bool check_errors);
456 void (*suspend_all_sms)(struct gk20a *g,
457 u32 global_esr_mask, bool check_errors);
458 void (*resume_single_sm)(struct gk20a *g,
459 u32 gpc, u32 tpc, u32 sm);
460 void (*resume_all_sms)(struct gk20a *g);
461 void (*disable_rd_coalesce)(struct gk20a *g);
462 void (*init_ctxsw_hdr_data)(struct gk20a *g,
463 struct nvgpu_mem *mem);
464 void (*init_gfxp_wfi_timeout_count)(struct gk20a *g);
465 unsigned long (*get_max_gfxp_wfi_timeout_count)
466 (struct gk20a *g);
467 void (*ecc_init_scrub_reg)(struct gk20a *g);
468 u32 (*get_gpcs_swdx_dss_zbc_c_format_reg)(struct gk20a *g);
469 u32 (*get_gpcs_swdx_dss_zbc_z_format_reg)(struct gk20a *g);
470 void (*dump_ctxsw_stats)(struct gk20a *g, struct vm_gk20a *vm,
471 struct nvgpu_gr_ctx *gr_ctx);
472 void (*fecs_host_int_enable)(struct gk20a *g);
473 int (*handle_ssync_hww)(struct gk20a *g);
474 int (*handle_notify_pending)(struct gk20a *g,
475 struct gr_gk20a_isr_data *isr_data);
476 int (*handle_semaphore_pending)(struct gk20a *g,
477 struct gr_gk20a_isr_data *isr_data);
478 int (*add_ctxsw_reg_pm_fbpa)(struct gk20a *g,
479 struct ctxsw_buf_offset_map_entry *map,
480 struct aiv_list_gk20a *regs,
481 u32 *count, u32 *offset,
482 u32 max_cnt, u32 base,
483 u32 num_fbpas, u32 stride, u32 mask);
484 int (*add_ctxsw_reg_perf_pma)(struct ctxsw_buf_offset_map_entry *map,
485 struct aiv_list_gk20a *regs,
486 u32 *count, u32 *offset,
487 u32 max_cnt, u32 base, u32 mask);
488 int (*decode_priv_addr)(struct gk20a *g, u32 addr,
489 enum ctxsw_addr_type *addr_type,
490 u32 *gpc_num, u32 *tpc_num,
491 u32 *ppc_num, u32 *be_num,
492 u32 *broadcast_flags);
493 int (*create_priv_addr_table)(struct gk20a *g,
494 u32 addr,
495 u32 *priv_addr_table,
496 u32 *num_registers);
497 u32 (*get_pmm_per_chiplet_offset)(void);
498 void (*split_fbpa_broadcast_addr)(struct gk20a *g, u32 addr,
499 u32 num_fbpas,
500 u32 *priv_addr_table,
501 u32 *priv_addr_table_index);
502 u32 (*fecs_ctxsw_mailbox_size)(void);
503 int (*init_sw_bundle64)(struct gk20a *g);
504 int (*alloc_global_ctx_buffers)(struct gk20a *g);
505 int (*map_global_ctx_buffers)(struct gk20a *g,
506 struct channel_gk20a *c);
507 int (*commit_global_ctx_buffers)(struct gk20a *g,
508 struct channel_gk20a *c, bool patch);
509 u32 (*get_nonpes_aware_tpc)(struct gk20a *g, u32 gpc, u32 tpc);
510 int (*get_offset_in_gpccs_segment)(struct gk20a *g,
511 enum ctxsw_addr_type addr_type, u32 num_tpcs,
512 u32 num_ppcs, u32 reg_list_ppc_count,
513 u32 *__offset_in_segment);
514 void (*set_debug_mode)(struct gk20a *g, bool enable);
515 } gr;
516 struct {
517 void (*init_hw)(struct gk20a *g);
518 void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr);
519 void (*init_fs_state)(struct gk20a *g);
520 void (*reset)(struct gk20a *g);
521 void (*init_uncompressed_kind_map)(struct gk20a *g);
522 void (*init_kind_attr)(struct gk20a *g);
523 void (*set_mmu_page_size)(struct gk20a *g);
524 bool (*set_use_full_comp_tag_line)(struct gk20a *g);
525 u32 (*mmu_ctrl)(struct gk20a *g);
526 u32 (*mmu_debug_ctrl)(struct gk20a *g);
527 u32 (*mmu_debug_wr)(struct gk20a *g);
528 u32 (*mmu_debug_rd)(struct gk20a *g);
529
530 /*
531 * Compression tag line coverage. When mapping a compressible
532 * buffer, ctagline is increased when the virtual address
533 * crosses over the compression page boundary.
534 */
535 unsigned int (*compression_page_size)(struct gk20a *g);
536
537 /*
538 * Minimum page size that can be used for compressible kinds.
539 */
540 unsigned int (*compressible_page_size)(struct gk20a *g);
541
542 /*
543 * Compressible kind mappings: Mask for the virtual and physical
544 * address bits that must match.
545 */
546 u32 (*compression_align_mask)(struct gk20a *g);
547
548 void (*dump_vpr_wpr_info)(struct gk20a *g);
549 int (*vpr_info_fetch)(struct gk20a *g);
550 void (*read_wpr_info)(struct gk20a *g,
551 struct wpr_carveout_info *inf);
552 bool (*is_debug_mode_enabled)(struct gk20a *g);
553 void (*set_debug_mode)(struct gk20a *g, bool enable);
554 int (*tlb_invalidate)(struct gk20a *g, struct nvgpu_mem *pdb);
555 void (*hub_isr)(struct gk20a *g);
556 void (*handle_replayable_fault)(struct gk20a *g);
557 int (*mem_unlock)(struct gk20a *g);
558 int (*init_nvlink)(struct gk20a *g);
559 int (*enable_nvlink)(struct gk20a *g);
560 void (*enable_hub_intr)(struct gk20a *g);
561 void (*disable_hub_intr)(struct gk20a *g);
562 int (*init_fbpa)(struct gk20a *g);
563 void (*fbpa_isr)(struct gk20a *g);
564 void (*write_mmu_fault_buffer_lo_hi)(struct gk20a *g, u32 index,
565 u32 addr_lo, u32 addr_hi);
566 void (*write_mmu_fault_buffer_get)(struct gk20a *g, u32 index,
567 u32 reg_val);
568 void (*write_mmu_fault_buffer_size)(struct gk20a *g, u32 index,
569 u32 reg_val);
570 void (*write_mmu_fault_status)(struct gk20a *g, u32 reg_val);
571 u32 (*read_mmu_fault_buffer_get)(struct gk20a *g, u32 index);
572 u32 (*read_mmu_fault_buffer_put)(struct gk20a *g, u32 index);
573 u32 (*read_mmu_fault_buffer_size)(struct gk20a *g, u32 index);
574 void (*read_mmu_fault_addr_lo_hi)(struct gk20a *g,
575 u32 *addr_lo, u32 *addr_hi);
576 void (*read_mmu_fault_inst_lo_hi)(struct gk20a *g,
577 u32 *inst_lo, u32 *inst_hi);
578 u32 (*read_mmu_fault_info)(struct gk20a *g);
579 u32 (*read_mmu_fault_status)(struct gk20a *g);
580 int (*mmu_invalidate_replay)(struct gk20a *g,
581 u32 invalidate_replay_val);
582 bool (*mmu_fault_pending)(struct gk20a *g);
583 bool (*is_fault_buf_enabled)(struct gk20a *g, u32 index);
584 void (*fault_buf_set_state_hw)(struct gk20a *g,
585 u32 index, u32 state);
586 void (*fault_buf_configure_hw)(struct gk20a *g, u32 index);
587 size_t (*get_vidmem_size)(struct gk20a *g);
588 } fb;
589 struct {
590 void (*slcg_bus_load_gating_prod)(struct gk20a *g, bool prod);
591 void (*slcg_ce2_load_gating_prod)(struct gk20a *g, bool prod);
592 void (*slcg_chiplet_load_gating_prod)(struct gk20a *g, bool prod);
593 void (*slcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod);
594 void (*slcg_fb_load_gating_prod)(struct gk20a *g, bool prod);
595 void (*slcg_fifo_load_gating_prod)(struct gk20a *g, bool prod);
596 void (*slcg_gr_load_gating_prod)(struct gk20a *g, bool prod);
597 void (*slcg_ltc_load_gating_prod)(struct gk20a *g, bool prod);
598 void (*slcg_perf_load_gating_prod)(struct gk20a *g, bool prod);
599 void (*slcg_priring_load_gating_prod)(struct gk20a *g, bool prod);
600 void (*slcg_pmu_load_gating_prod)(struct gk20a *g, bool prod);
601 void (*slcg_therm_load_gating_prod)(struct gk20a *g, bool prod);
602 void (*slcg_xbar_load_gating_prod)(struct gk20a *g, bool prod);
603 void (*blcg_bus_load_gating_prod)(struct gk20a *g, bool prod);
604 void (*blcg_ce_load_gating_prod)(struct gk20a *g, bool prod);
605 void (*blcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod);
606 void (*blcg_fb_load_gating_prod)(struct gk20a *g, bool prod);
607 void (*blcg_fifo_load_gating_prod)(struct gk20a *g, bool prod);
608 void (*blcg_gr_load_gating_prod)(struct gk20a *g, bool prod);
609 void (*blcg_ltc_load_gating_prod)(struct gk20a *g, bool prod);
610 void (*blcg_pwr_csb_load_gating_prod)(struct gk20a *g, bool prod);
611 void (*blcg_pmu_load_gating_prod)(struct gk20a *g, bool prod);
612 void (*blcg_xbar_load_gating_prod)(struct gk20a *g, bool prod);
613 void (*pg_gr_load_gating_prod)(struct gk20a *g, bool prod);
614 } clock_gating;
615 struct {
616 void (*post_events)(struct channel_gk20a *ch);
617 } debugger;
618 struct {
619 int (*setup_sw)(struct gk20a *g);
620 int (*init_fifo_setup_hw)(struct gk20a *g);
621 void (*bind_channel)(struct channel_gk20a *ch_gk20a);
622 void (*unbind_channel)(struct channel_gk20a *ch_gk20a);
623 void (*disable_channel)(struct channel_gk20a *ch);
624 void (*enable_channel)(struct channel_gk20a *ch);
625 int (*alloc_inst)(struct gk20a *g, struct channel_gk20a *ch);
626 void (*free_inst)(struct gk20a *g, struct channel_gk20a *ch);
627 int (*setup_ramfc)(struct channel_gk20a *c, u64 gpfifo_base,
628 u32 gpfifo_entries,
629 unsigned long acquire_timeout,
630 u32 flags);
631 int (*resetup_ramfc)(struct channel_gk20a *c);
632 int (*preempt_channel)(struct gk20a *g, u32 chid);
633 int (*preempt_tsg)(struct gk20a *g, u32 tsgid);
634 int (*enable_tsg)(struct tsg_gk20a *tsg);
635 int (*disable_tsg)(struct tsg_gk20a *tsg);
636 int (*tsg_verify_channel_status)(struct channel_gk20a *ch);
637 void (*tsg_verify_status_ctx_reload)(struct channel_gk20a *ch);
638 void (*tsg_verify_status_faulted)(struct channel_gk20a *ch);
639 int (*reschedule_runlist)(struct channel_gk20a *ch,
640 bool preempt_next);
641 int (*update_runlist)(struct gk20a *g, u32 runlist_id,
642 u32 chid, bool add,
643 bool wait_for_finish);
644 void (*trigger_mmu_fault)(struct gk20a *g,
645 unsigned long engine_ids);
646 void (*get_mmu_fault_info)(struct gk20a *g, u32 mmu_fault_id,
647 struct mmu_fault_info *mmfault);
648 void (*get_mmu_fault_desc)(struct mmu_fault_info *mmfault);
649 void (*get_mmu_fault_client_desc)(
650 struct mmu_fault_info *mmfault);
651 void (*get_mmu_fault_gpc_desc)(struct mmu_fault_info *mmfault);
652 void (*apply_pb_timeout)(struct gk20a *g);
653 void (*apply_ctxsw_timeout_intr)(struct gk20a *g);
654 int (*wait_engine_idle)(struct gk20a *g);
655 u32 (*get_num_fifos)(struct gk20a *g);
656 u32 (*get_pbdma_signature)(struct gk20a *g);
657 int (*set_runlist_interleave)(struct gk20a *g, u32 id,
658 u32 runlist_id,
659 u32 new_level);
660 int (*tsg_set_timeslice)(struct tsg_gk20a *tsg, u32 timeslice);
661 u32 (*default_timeslice_us)(struct gk20a *);
662 int (*force_reset_ch)(struct channel_gk20a *ch,
663 u32 err_code, bool verbose);
664 int (*engine_enum_from_type)(struct gk20a *g, u32 engine_type,
665 u32 *inst_id);
666 void (*device_info_data_parse)(struct gk20a *g,
667 u32 table_entry, u32 *inst_id,
668 u32 *pri_base, u32 *fault_id);
669 u32 (*device_info_fault_id)(u32 table_entry);
670 int (*tsg_bind_channel)(struct tsg_gk20a *tsg,
671 struct channel_gk20a *ch);
672 int (*tsg_unbind_channel)(struct channel_gk20a *ch);
673 int (*tsg_open)(struct tsg_gk20a *tsg);
674 void (*tsg_release)(struct tsg_gk20a *tsg);
675 u32 (*eng_runlist_base_size)(void);
676 int (*init_engine_info)(struct fifo_gk20a *f);
677 u32 (*runlist_entry_size)(void);
678 void (*get_tsg_runlist_entry)(struct tsg_gk20a *tsg,
679 u32 *runlist);
680 void (*get_ch_runlist_entry)(struct channel_gk20a *ch,
681 u32 *runlist);
682 u32 (*userd_gp_get)(struct gk20a *g, struct channel_gk20a *ch);
683 void (*userd_gp_put)(struct gk20a *g, struct channel_gk20a *ch);
684 u64 (*userd_pb_get)(struct gk20a *g, struct channel_gk20a *ch);
685 void (*free_channel_ctx_header)(struct channel_gk20a *ch);
686 bool (*is_fault_engine_subid_gpc)(struct gk20a *g,
687 u32 engine_subid);
688 void (*dump_pbdma_status)(struct gk20a *g,
689 struct gk20a_debug_output *o);
690 void (*dump_eng_status)(struct gk20a *g,
691 struct gk20a_debug_output *o);
692 void (*dump_channel_status_ramfc)(struct gk20a *g,
693 struct gk20a_debug_output *o, u32 chid,
694 struct ch_state *ch_state);
695 u32 (*intr_0_error_mask)(struct gk20a *g);
696 int (*is_preempt_pending)(struct gk20a *g, u32 id,
697 unsigned int id_type);
698 void (*init_pbdma_intr_descs)(struct fifo_gk20a *f);
699 int (*reset_enable_hw)(struct gk20a *g);
700 int (*setup_userd)(struct channel_gk20a *c);
701 u32 (*pbdma_acquire_val)(u64 timeout);
702 void (*teardown_ch_tsg)(struct gk20a *g, u32 act_eng_bitmask,
703 u32 id, unsigned int id_type, unsigned int rc_type,
704 struct mmu_fault_info *mmfault);
705 bool (*handle_sched_error)(struct gk20a *g);
706 bool (*handle_ctxsw_timeout)(struct gk20a *g, u32 fifo_intr);
707 unsigned int (*handle_pbdma_intr_0)(struct gk20a *g,
708 u32 pbdma_id, u32 pbdma_intr_0,
709 u32 *handled, u32 *error_notifier);
710 unsigned int (*handle_pbdma_intr_1)(struct gk20a *g,
711 u32 pbdma_id, u32 pbdma_intr_1,
712 u32 *handled, u32 *error_notifier);
713 void (*init_eng_method_buffers)(struct gk20a *g,
714 struct tsg_gk20a *tsg);
715 void (*deinit_eng_method_buffers)(struct gk20a *g,
716 struct tsg_gk20a *tsg);
717 u32 (*get_preempt_timeout)(struct gk20a *g);
718 void (*post_event_id)(struct tsg_gk20a *tsg, int event_id);
719 void (*ch_abort_clean_up)(struct channel_gk20a *ch);
720 bool (*check_tsg_ctxsw_timeout)(struct tsg_gk20a *tsg,
721 bool *verbose, u32 *ms);
722 bool (*check_ch_ctxsw_timeout)(struct channel_gk20a *ch,
723 bool *verbose, u32 *ms);
724 int (*channel_suspend)(struct gk20a *g);
725 int (*channel_resume)(struct gk20a *g);
726 void (*set_error_notifier)(struct channel_gk20a *ch, u32 error);
727#ifdef CONFIG_TEGRA_GK20A_NVHOST
728 int (*alloc_syncpt_buf)(struct channel_gk20a *c,
729 u32 syncpt_id, struct nvgpu_mem *syncpt_buf);
730 void (*free_syncpt_buf)(struct channel_gk20a *c,
731 struct nvgpu_mem *syncpt_buf);
732 void (*add_syncpt_wait_cmd)(struct gk20a *g,
733 struct priv_cmd_entry *cmd, u32 off,
734 u32 id, u32 thresh, u64 gpu_va);
735 u32 (*get_syncpt_wait_cmd_size)(void);
736 void (*add_syncpt_incr_cmd)(struct gk20a *g,
737 bool wfi_cmd, struct priv_cmd_entry *cmd,
738 u32 id, u64 gpu_va);
739 u32 (*get_syncpt_incr_cmd_size)(bool wfi_cmd);
740 int (*get_sync_ro_map)(struct vm_gk20a *vm,
741 u64 *base_gpuva, u32 *sync_size);
742 u32 (*get_syncpt_incr_per_release)(void);
743#endif
744 void (*runlist_hw_submit)(struct gk20a *g, u32 runlist_id,
745 u32 count, u32 buffer_index);
746 int (*runlist_wait_pending)(struct gk20a *g, u32 runlist_id);
747 void (*ring_channel_doorbell)(struct channel_gk20a *c);
748 u32 (*get_sema_wait_cmd_size)(void);
749 u32 (*get_sema_incr_cmd_size)(void);
750 void (*add_sema_cmd)(struct gk20a *g,
751 struct nvgpu_semaphore *s, u64 sema_va,
752 struct priv_cmd_entry *cmd,
753 u32 off, bool acquire, bool wfi);
754 } fifo;
755 struct pmu_v {
756 u32 (*get_pmu_cmdline_args_size)(struct nvgpu_pmu *pmu);
757 void (*set_pmu_cmdline_args_cpu_freq)(struct nvgpu_pmu *pmu,
758 u32 freq);
759 void (*set_pmu_cmdline_args_trace_size)(struct nvgpu_pmu *pmu,
760 u32 size);
761 void (*set_pmu_cmdline_args_trace_dma_base)(
762 struct nvgpu_pmu *pmu);
763 void (*config_pmu_cmdline_args_super_surface)(
764 struct nvgpu_pmu *pmu);
765 void (*set_pmu_cmdline_args_trace_dma_idx)(
766 struct nvgpu_pmu *pmu, u32 idx);
767 void * (*get_pmu_cmdline_args_ptr)(struct nvgpu_pmu *pmu);
768 u32 (*get_pmu_allocation_struct_size)(struct nvgpu_pmu *pmu);
769 void (*set_pmu_allocation_ptr)(struct nvgpu_pmu *pmu,
770 void **pmu_alloc_ptr, void *assign_ptr);
771 void (*pmu_allocation_set_dmem_size)(struct nvgpu_pmu *pmu,
772 void *pmu_alloc_ptr, u16 size);
773 u16 (*pmu_allocation_get_dmem_size)(struct nvgpu_pmu *pmu,
774 void *pmu_alloc_ptr);
775 u32 (*pmu_allocation_get_dmem_offset)(struct nvgpu_pmu *pmu,
776 void *pmu_alloc_ptr);
777 u32 * (*pmu_allocation_get_dmem_offset_addr)(
778 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr);
779 void (*pmu_allocation_set_dmem_offset)(struct nvgpu_pmu *pmu,
780 void *pmu_alloc_ptr, u32 offset);
781 void * (*pmu_allocation_get_fb_addr)(
782 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr);
783 u32 (*pmu_allocation_get_fb_size)(
784 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr);
785 void (*get_pmu_init_msg_pmu_queue_params)(
786 struct nvgpu_falcon_queue *queue, u32 id,
787 void *pmu_init_msg);
788 void *(*get_pmu_msg_pmu_init_msg_ptr)(
789 struct pmu_init_msg *init);
790 u16 (*get_pmu_init_msg_pmu_sw_mg_off)(
791 union pmu_init_msg_pmu *init_msg);
792 u16 (*get_pmu_init_msg_pmu_sw_mg_size)(
793 union pmu_init_msg_pmu *init_msg);
794 u32 (*get_pmu_perfmon_cmd_start_size)(void);
795 int (*get_perfmon_cmd_start_offsetofvar)(
796 enum pmu_perfmon_cmd_start_fields field);
797 void (*perfmon_start_set_cmd_type)(struct pmu_perfmon_cmd *pc,
798 u8 value);
799 void (*perfmon_start_set_group_id)(struct pmu_perfmon_cmd *pc,
800 u8 value);
801 void (*perfmon_start_set_state_id)(struct pmu_perfmon_cmd *pc,
802 u8 value);
803 void (*perfmon_start_set_flags)(struct pmu_perfmon_cmd *pc,
804 u8 value);
805 u8 (*perfmon_start_get_flags)(struct pmu_perfmon_cmd *pc);
806 u32 (*get_pmu_perfmon_cmd_init_size)(void);
807 int (*get_perfmon_cmd_init_offsetofvar)(
808 enum pmu_perfmon_cmd_start_fields field);
809 void (*perfmon_cmd_init_set_sample_buffer)(
810 struct pmu_perfmon_cmd *pc, u16 value);
811 void (*perfmon_cmd_init_set_dec_cnt)(
812 struct pmu_perfmon_cmd *pc, u8 value);
813 void (*perfmon_cmd_init_set_base_cnt_id)(
814 struct pmu_perfmon_cmd *pc, u8 value);
815 void (*perfmon_cmd_init_set_samp_period_us)(
816 struct pmu_perfmon_cmd *pc, u32 value);
817 void (*perfmon_cmd_init_set_num_cnt)(struct pmu_perfmon_cmd *pc,
818 u8 value);
819 void (*perfmon_cmd_init_set_mov_avg)(struct pmu_perfmon_cmd *pc,
820 u8 value);
821 void *(*get_pmu_seq_in_a_ptr)(
822 struct pmu_sequence *seq);
823 void *(*get_pmu_seq_out_a_ptr)(
824 struct pmu_sequence *seq);
825 void (*set_pmu_cmdline_args_secure_mode)(struct nvgpu_pmu *pmu,
826 u32 val);
827 u32 (*get_perfmon_cntr_sz)(struct nvgpu_pmu *pmu);
828 void * (*get_perfmon_cntr_ptr)(struct nvgpu_pmu *pmu);
829 void (*set_perfmon_cntr_ut)(struct nvgpu_pmu *pmu, u16 ut);
830 void (*set_perfmon_cntr_lt)(struct nvgpu_pmu *pmu, u16 lt);
831 void (*set_perfmon_cntr_valid)(struct nvgpu_pmu *pmu, u8 val);
832 void (*set_perfmon_cntr_index)(struct nvgpu_pmu *pmu, u8 val);
833 void (*set_perfmon_cntr_group_id)(struct nvgpu_pmu *pmu,
834 u8 gid);
835
836 u8 (*pg_cmd_eng_buf_load_size)(struct pmu_pg_cmd *pg);
837 void (*pg_cmd_eng_buf_load_set_cmd_type)(struct pmu_pg_cmd *pg,
838 u8 value);
839 void (*pg_cmd_eng_buf_load_set_engine_id)(struct pmu_pg_cmd *pg,
840 u8 value);
841 void (*pg_cmd_eng_buf_load_set_buf_idx)(struct pmu_pg_cmd *pg,
842 u8 value);
843 void (*pg_cmd_eng_buf_load_set_pad)(struct pmu_pg_cmd *pg,
844 u8 value);
845 void (*pg_cmd_eng_buf_load_set_buf_size)(struct pmu_pg_cmd *pg,
846 u16 value);
847 void (*pg_cmd_eng_buf_load_set_dma_base)(struct pmu_pg_cmd *pg,
848 u32 value);
849 void (*pg_cmd_eng_buf_load_set_dma_offset)(struct pmu_pg_cmd *pg,
850 u8 value);
851 void (*pg_cmd_eng_buf_load_set_dma_idx)(struct pmu_pg_cmd *pg,
852 u8 value);
853 struct {
854 int (*boardobjgrp_pmucmd_construct_impl)
855 (struct gk20a *g,
856 struct boardobjgrp *pboardobjgrp,
857 struct boardobjgrp_pmu_cmd *cmd, u8 id, u8 msgid,
858 u8 hdrsize, u8 entrysize, u16 fbsize, u32 ss_offset,
859 u8 rpc_func_id);
860 int (*boardobjgrp_pmuset_impl)(struct gk20a *g,
861 struct boardobjgrp *pboardobjgrp);
862 int (*boardobjgrp_pmugetstatus_impl)(struct gk20a *g,
863 struct boardobjgrp *pboardobjgrp,
864 struct boardobjgrpmask *mask);
865 int (*is_boardobjgrp_pmucmd_id_valid)(struct gk20a *g,
866 struct boardobjgrp *pboardobjgrp,
867 struct boardobjgrp_pmu_cmd *cmd);
868 } boardobj;
869 struct {
870 u32 (*volt_set_voltage)(struct gk20a *g,
871 u32 logic_voltage_uv, u32 sram_voltage_uv);
872 u32 (*volt_get_voltage)(struct gk20a *g,
873 u8 volt_domain, u32 *pvoltage_uv);
874 u32 (*volt_send_load_cmd_to_pmu)(struct gk20a *g);
875 } volt;
876 struct {
877 u32 (*get_vbios_clk_domain)(u32 vbios_domain);
878 u32 (*clk_avfs_get_vin_cal_data)(struct gk20a *g,
879 struct avfsvinobjs *pvinobjs,
880 struct vin_device_v20 *pvindev);
881 u32 (*clk_vf_change_inject_data_fill)(struct gk20a *g,
882 struct nv_pmu_clk_rpc *rpccall,
883 struct set_fll_clk *setfllclk);
884 u32 (*perf_pmu_vfe_load)(struct gk20a *g);
885 u32 (*clk_set_boot_clk)(struct gk20a *g);
886 }clk;
887 } pmu_ver;
888 struct {
889 int (*get_netlist_name)(struct gk20a *g, int index, char *name);
890 bool (*is_fw_defined)(void);
891 } gr_ctx;
892#ifdef CONFIG_GK20A_CTXSW_TRACE
893 /*
894 * Currently only supported on Linux due to the extremely tight
895 * integration with Linux device driver structure (in particular
896 * mmap).
897 */
898 struct {
899 int (*init)(struct gk20a *g);
900 int (*max_entries)(struct gk20a *,
901 struct nvgpu_gpu_ctxsw_trace_filter *filter);
902 int (*flush)(struct gk20a *g);
903 int (*poll)(struct gk20a *g);
904 int (*enable)(struct gk20a *g);
905 int (*disable)(struct gk20a *g);
906 bool (*is_enabled)(struct gk20a *g);
907 int (*reset)(struct gk20a *g);
908 int (*bind_channel)(struct gk20a *g, struct channel_gk20a *ch);
909 int (*unbind_channel)(struct gk20a *g,
910 struct channel_gk20a *ch);
911 int (*deinit)(struct gk20a *g);
912 int (*alloc_user_buffer)(struct gk20a *g,
913 void **buf, size_t *size);
914 int (*free_user_buffer)(struct gk20a *g);
915 int (*mmap_user_buffer)(struct gk20a *g,
916 struct vm_area_struct *vma);
917 int (*set_filter)(struct gk20a *g,
918 struct nvgpu_gpu_ctxsw_trace_filter *filter);
919 } fecs_trace;
920#endif
921 struct {
922 bool (*support_sparse)(struct gk20a *g);
923 u64 (*gmmu_map)(struct vm_gk20a *vm,
924 u64 map_offset,
925 struct nvgpu_sgt *sgt,
926 u64 buffer_offset,
927 u64 size,
928 u32 pgsz_idx,
929 u8 kind_v,
930 u32 ctag_offset,
931 u32 flags,
932 enum gk20a_mem_rw_flag rw_flag,
933 bool clear_ctags,
934 bool sparse,
935 bool priv,
936 struct vm_gk20a_mapping_batch *batch,
937 enum nvgpu_aperture aperture);
938 void (*gmmu_unmap)(struct vm_gk20a *vm,
939 u64 vaddr,
940 u64 size,
941 u32 pgsz_idx,
942 bool va_allocated,
943 enum gk20a_mem_rw_flag rw_flag,
944 bool sparse,
945 struct vm_gk20a_mapping_batch *batch);
946 int (*vm_bind_channel)(struct vm_gk20a *vm,
947 struct channel_gk20a *ch);
948 int (*fb_flush)(struct gk20a *g);
949 void (*l2_invalidate)(struct gk20a *g);
950 void (*l2_flush)(struct gk20a *g, bool invalidate);
951 void (*cbc_clean)(struct gk20a *g);
952 void (*set_big_page_size)(struct gk20a *g,
953 struct nvgpu_mem *mem, int size);
954 u32 (*get_big_page_sizes)(void);
955 u32 (*get_default_big_page_size)(void);
956 u32 (*get_iommu_bit)(struct gk20a *g);
957 int (*init_mm_setup_hw)(struct gk20a *g);
958 bool (*is_bar1_supported)(struct gk20a *g);
959 int (*init_bar2_vm)(struct gk20a *g);
960 void (*remove_bar2_vm)(struct gk20a *g);
961 const struct gk20a_mmu_level *
962 (*get_mmu_levels)(struct gk20a *g, u32 big_page_size);
963 void (*init_pdb)(struct gk20a *g, struct nvgpu_mem *inst_block,
964 struct vm_gk20a *vm);
965 u64 (*gpu_phys_addr)(struct gk20a *g,
966 struct nvgpu_gmmu_attrs *attrs, u64 phys);
967 int (*alloc_inst_block)(struct gk20a *g,
968 struct nvgpu_mem *inst_block);
969 void (*init_inst_block)(struct nvgpu_mem *inst_block,
970 struct vm_gk20a *vm, u32 big_page_size);
971 bool (*mmu_fault_pending)(struct gk20a *g);
972 void (*fault_info_mem_destroy)(struct gk20a *g);
973 void (*mmu_fault_disable_hw)(struct gk20a *g);
974 u32 (*get_kind_invalid)(void);
975 u32 (*get_kind_pitch)(void);
976 u32 (*get_flush_retries)(struct gk20a *g,
977 enum nvgpu_flush_op op);
978 } mm;
979 /*
980 * This function is called to allocate secure memory (memory
981 * that the CPU cannot see). The function should fill the
982 * context buffer descriptor (especially fields destroy, sgt,
983 * size).
984 */
985 int (*secure_alloc)(struct gk20a *g,
986 struct gr_ctx_buffer_desc *desc,
987 size_t size);
988 struct {
989 void (*exit)(struct gk20a *g, struct nvgpu_mem *mem,
990 struct nvgpu_sgl *sgl);
991 u32 (*data032_r)(u32 i);
992 } pramin;
993 struct {
994 int (*init_therm_setup_hw)(struct gk20a *g);
995 void (*init_elcg_mode)(struct gk20a *g, u32 mode, u32 engine);
996 void (*init_blcg_mode)(struct gk20a *g, u32 mode, u32 engine);
997 int (*elcg_init_idle_filters)(struct gk20a *g);
998#ifdef CONFIG_DEBUG_FS
999 void (*therm_debugfs_init)(struct gk20a *g);
1000#endif
1001 int (*get_internal_sensor_curr_temp)(struct gk20a *g, u32 *temp_f24_8);
1002 void (*get_internal_sensor_limits)(s32 *max_24_8,
1003 s32 *min_24_8);
1004 u32 (*configure_therm_alert)(struct gk20a *g, s32 curr_warn_temp);
1005 } therm;
1006 struct {
1007 bool (*is_pmu_supported)(struct gk20a *g);
1008 int (*prepare_ucode)(struct gk20a *g);
1009 int (*pmu_setup_hw_and_bootstrap)(struct gk20a *g);
1010 int (*pmu_nsbootstrap)(struct nvgpu_pmu *pmu);
1011 int (*pmu_init_perfmon)(struct nvgpu_pmu *pmu);
1012 int (*pmu_perfmon_start_sampling)(struct nvgpu_pmu *pmu);
1013 int (*pmu_perfmon_stop_sampling)(struct nvgpu_pmu *pmu);
1014 int (*pmu_perfmon_get_samples_rpc)(struct nvgpu_pmu *pmu);
1015 int (*pmu_setup_elpg)(struct gk20a *g);
1016 u32 (*pmu_get_queue_head)(u32 i);
1017 u32 (*pmu_get_queue_head_size)(void);
1018 u32 (*pmu_get_queue_tail_size)(void);
1019 u32 (*pmu_get_queue_tail)(u32 i);
1020 int (*pmu_queue_head)(struct gk20a *g,
1021 struct nvgpu_falcon_queue *queue, u32 *head, bool set);
1022 int (*pmu_queue_tail)(struct gk20a *g,
1023 struct nvgpu_falcon_queue *queue, u32 *tail, bool set);
1024 void (*pmu_msgq_tail)(struct nvgpu_pmu *pmu,
1025 u32 *tail, bool set);
1026 u32 (*pmu_mutex_size)(void);
1027 int (*pmu_mutex_acquire)(struct nvgpu_pmu *pmu,
1028 u32 id, u32 *token);
1029 int (*pmu_mutex_release)(struct nvgpu_pmu *pmu,
1030 u32 id, u32 *token);
1031 bool (*pmu_is_interrupted)(struct nvgpu_pmu *pmu);
1032 void (*pmu_isr)(struct gk20a *g);
1033 void (*pmu_init_perfmon_counter)(struct gk20a *g);
1034 void (*pmu_pg_idle_counter_config)(struct gk20a *g, u32 pg_engine_id);
1035 u32 (*pmu_read_idle_counter)(struct gk20a *g, u32 counter_id);
1036 void (*pmu_reset_idle_counter)(struct gk20a *g, u32 counter_id);
1037 void (*pmu_dump_elpg_stats)(struct nvgpu_pmu *pmu);
1038 void (*pmu_dump_falcon_stats)(struct nvgpu_pmu *pmu);
1039 void (*pmu_enable_irq)(struct nvgpu_pmu *pmu, bool enable);
1040 int (*init_wpr_region)(struct gk20a *g);
1041 int (*load_lsfalcon_ucode)(struct gk20a *g, u32 falconidmask);
1042 void (*write_dmatrfbase)(struct gk20a *g, u32 addr);
1043 void (*pmu_elpg_statistics)(struct gk20a *g, u32 pg_engine_id,
1044 struct pmu_pg_stats_data *pg_stat_data);
1045 int (*pmu_pg_init_param)(struct gk20a *g, u32 pg_engine_id);
1046 int (*pmu_pg_set_sub_feature_mask)(struct gk20a *g,
1047 u32 pg_engine_id);
1048 u32 (*pmu_pg_supported_engines_list)(struct gk20a *g);
1049 u32 (*pmu_pg_engines_feature_list)(struct gk20a *g,
1050 u32 pg_engine_id);
1051 bool (*pmu_is_lpwr_feature_supported)(struct gk20a *g,
1052 u32 feature_id);
1053 int (*pmu_lpwr_enable_pg)(struct gk20a *g, bool pstate_lock);
1054 int (*pmu_lpwr_disable_pg)(struct gk20a *g, bool pstate_lock);
1055 u32 (*pmu_pg_param_post_init)(struct gk20a *g);
1056 void (*dump_secure_fuses)(struct gk20a *g);
1057 int (*reset_engine)(struct gk20a *g, bool do_reset);
1058 bool (*is_engine_in_reset)(struct gk20a *g);
1059 int (*falcon_wait_for_halt)(struct gk20a *g,
1060 unsigned int timeout);
1061 int (*falcon_clear_halt_interrupt_status)(struct gk20a *g,
1062 unsigned int timeout);
1063 int (*init_falcon_setup_hw)(struct gk20a *g,
1064 void *desc, u32 bl_sz);
1065 bool (*is_lazy_bootstrap)(u32 falcon_id);
1066 bool (*is_priv_load)(u32 falcon_id);
1067 void (*get_wpr)(struct gk20a *g, struct wpr_carveout_info *inf);
1068 int (*alloc_blob_space)(struct gk20a *g,
1069 size_t size, struct nvgpu_mem *mem);
1070 int (*pmu_populate_loader_cfg)(struct gk20a *g,
1071 void *lsfm, u32 *p_bl_gen_desc_size);
1072 int (*flcn_populate_bl_dmem_desc)(struct gk20a *g,
1073 void *lsfm, u32 *p_bl_gen_desc_size, u32 falconid);
1074 void (*handle_ext_irq)(struct gk20a *g, u32 intr);
1075 void (*set_irqmask)(struct gk20a *g);
1076 void (*update_lspmu_cmdline_args)(struct gk20a *g);
1077 void (*setup_apertures)(struct gk20a *g);
1078 u32 (*get_irqdest)(struct gk20a *g);
1079 int (*alloc_super_surface)(struct gk20a *g,
1080 struct nvgpu_mem *super_surface, u32 size);
1081 bool (*is_debug_mode_enabled)(struct gk20a *g);
1082 } pmu;
1083 struct {
1084 int (*init_debugfs)(struct gk20a *g);
1085 void (*disable_slowboot)(struct gk20a *g);
1086 int (*init_clk_support)(struct gk20a *g);
1087 int (*suspend_clk_support)(struct gk20a *g);
1088 u32 (*get_crystal_clk_hz)(struct gk20a *g);
1089 int (*clk_domain_get_f_points)(struct gk20a *g,
1090 u32 clkapidomain, u32 *pfpointscount,
1091 u16 *pfreqpointsinmhz);
1092 unsigned long (*measure_freq)(struct gk20a *g, u32 api_domain);
1093 u32 (*get_rate_cntr)(struct gk20a *g, struct namemap_cfg *c);
1094 unsigned long (*get_rate)(struct gk20a *g, u32 api_domain);
1095 int (*set_rate)(struct gk20a *g, u32 api_domain, unsigned long rate);
1096 unsigned long (*get_fmax_at_vmin_safe)(struct gk20a *g);
1097 u32 (*get_ref_clock_rate)(struct gk20a *g);
1098 int (*predict_mv_at_hz_cur_tfloor)(struct clk_gk20a *clk,
1099 unsigned long rate);
1100 unsigned long (*get_maxrate)(struct gk20a *g, u32 api_domain);
1101 int (*prepare_enable)(struct clk_gk20a *clk);
1102 void (*disable_unprepare)(struct clk_gk20a *clk);
1103 int (*get_voltage)(struct clk_gk20a *clk, u64 *val);
1104 int (*get_gpcclk_clock_counter)(struct clk_gk20a *clk, u64 *val);
1105 int (*pll_reg_write)(struct gk20a *g, u32 reg, u32 val);
1106 int (*get_pll_debug_data)(struct gk20a *g,
1107 struct nvgpu_clk_pll_debug_data *d);
1108 int (*mclk_init)(struct gk20a *g);
1109 void (*mclk_deinit)(struct gk20a *g);
1110 int (*mclk_change)(struct gk20a *g, u16 val);
1111 bool split_rail_support;
1112 bool support_clk_freq_controller;
1113 bool support_pmgr_domain;
1114 bool support_lpwr_pg;
1115 } clk;
1116 struct {
1117 int (*arbiter_clk_init)(struct gk20a *g);
1118 u32 (*get_arbiter_clk_domains)(struct gk20a *g);
1119 int (*get_arbiter_clk_range)(struct gk20a *g, u32 api_domain,
1120 u16 *min_mhz, u16 *max_mhz);
1121 int (*get_arbiter_clk_default)(struct gk20a *g, u32 api_domain,
1122 u16 *default_mhz);
1123 void (*clk_arb_run_arbiter_cb)(struct nvgpu_clk_arb *arb);
1124 /* This function is inherently unsafe to call while
1125 * arbiter is running arbiter must be blocked
1126 * before calling this function */
1127 int (*get_current_pstate)(struct gk20a *g);
1128 void (*clk_arb_cleanup)(struct nvgpu_clk_arb *arb);
1129 } clk_arb;
1130 struct {
1131 int (*handle_pmu_perf_event)(struct gk20a *g, void *pmu_msg);
1132 } perf;
1133 struct {
1134 int (*exec_regops)(struct dbg_session_gk20a *dbg_s,
1135 struct nvgpu_dbg_reg_op *ops,
1136 u64 num_ops,
1137 bool *is_current_ctx);
1138 const struct regop_offset_range* (
1139 *get_global_whitelist_ranges)(void);
1140 u64 (*get_global_whitelist_ranges_count)(void);
1141 const struct regop_offset_range* (
1142 *get_context_whitelist_ranges)(void);
1143 u64 (*get_context_whitelist_ranges_count)(void);
1144 const u32* (*get_runcontrol_whitelist)(void);
1145 u64 (*get_runcontrol_whitelist_count)(void);
1146 const u32* (*get_qctl_whitelist)(void);
1147 u64 (*get_qctl_whitelist_count)(void);
1148 } regops;
1149 struct {
1150 void (*intr_mask)(struct gk20a *g);
1151 void (*intr_enable)(struct gk20a *g);
1152 void (*intr_unit_config)(struct gk20a *g,
1153 bool enable, bool is_stalling, u32 unit);
1154 void (*isr_stall)(struct gk20a *g);
1155 bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr);
1156 bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr);
1157 bool (*is_stall_and_eng_intr_pending)(struct gk20a *g,
1158 u32 act_eng_id, u32 *eng_intr_pending);
1159 u32 (*intr_stall)(struct gk20a *g);
1160 void (*intr_stall_pause)(struct gk20a *g);
1161 void (*intr_stall_resume)(struct gk20a *g);
1162 u32 (*intr_nonstall)(struct gk20a *g);
1163 void (*intr_nonstall_pause)(struct gk20a *g);
1164 void (*intr_nonstall_resume)(struct gk20a *g);
1165 u32 (*isr_nonstall)(struct gk20a *g);
1166 void (*enable)(struct gk20a *g, u32 units);
1167 void (*disable)(struct gk20a *g, u32 units);
1168 void (*reset)(struct gk20a *g, u32 units);
1169 u32 (*boot_0)(struct gk20a *g, u32 *arch, u32 *impl, u32 *rev);
1170 bool (*is_intr1_pending)(struct gk20a *g, enum nvgpu_unit unit, u32 mc_intr_1);
1171 void (*log_pending_intrs)(struct gk20a *g);
1172 } mc;
1173 struct {
1174 void (*show_dump)(struct gk20a *g,
1175 struct gk20a_debug_output *o);
1176 } debug;
1177 struct {
1178 int (*dbg_set_powergate)(struct dbg_session_gk20a *dbg_s,
1179 bool disable_powergate);
1180 bool (*check_and_set_global_reservation)(
1181 struct dbg_session_gk20a *dbg_s,
1182 struct dbg_profiler_object_data *prof_obj);
1183 bool (*check_and_set_context_reservation)(
1184 struct dbg_session_gk20a *dbg_s,
1185 struct dbg_profiler_object_data *prof_obj);
1186 void (*release_profiler_reservation)(
1187 struct dbg_session_gk20a *dbg_s,
1188 struct dbg_profiler_object_data *prof_obj);
1189 int (*perfbuffer_enable)(struct gk20a *g, u64 offset, u32 size);
1190 int (*perfbuffer_disable)(struct gk20a *g);
1191 } dbg_session_ops;
1192
1193 int (*get_litter_value)(struct gk20a *g, int value);
1194 int (*chip_init_gpu_characteristics)(struct gk20a *g);
1195
1196 struct {
1197 void (*init_hw)(struct gk20a *g);
1198 void (*isr)(struct gk20a *g);
1199 int (*bar1_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst);
1200 int (*bar2_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst);
1201 u32 (*set_bar0_window)(struct gk20a *g, struct nvgpu_mem *mem,
1202 struct nvgpu_sgt *sgt, struct nvgpu_sgl *sgl,
1203 u32 w);
1204 u32 (*read_sw_scratch)(struct gk20a *g, u32 index);
1205 void (*write_sw_scratch)(struct gk20a *g, u32 index, u32 val);
1206 } bus;
1207
1208 struct {
1209 void (*isr)(struct gk20a *g);
1210 int (*read_ptimer)(struct gk20a *g, u64 *value);
1211 int (*get_timestamps_zipper)(struct gk20a *g,
1212 u32 source_id, u32 count,
1213 struct nvgpu_cpu_time_correlation_sample *);
1214 } ptimer;
1215
1216 struct {
1217 int (*init)(struct gk20a *g);
1218 int (*preos_wait_for_halt)(struct gk20a *g);
1219 void (*preos_reload_check)(struct gk20a *g);
1220 int (*devinit)(struct gk20a *g);
1221 int (*preos)(struct gk20a *g);
1222 int (*verify_devinit)(struct gk20a *g);
1223 } bios;
1224
1225#if defined(CONFIG_GK20A_CYCLE_STATS)
1226 struct {
1227 int (*enable_snapshot)(struct channel_gk20a *ch,
1228 struct gk20a_cs_snapshot_client *client);
1229 void (*disable_snapshot)(struct gr_gk20a *gr);
1230 int (*check_data_available)(struct channel_gk20a *ch,
1231 u32 *pending,
1232 bool *hw_overflow);
1233 void (*set_handled_snapshots)(struct gk20a *g, u32 num);
1234 u32 (*allocate_perfmon_ids)(struct gk20a_cs_snapshot *data,
1235 u32 count);
1236 u32 (*release_perfmon_ids)(struct gk20a_cs_snapshot *data,
1237 u32 start,
1238 u32 count);
1239 int (*detach_snapshot)(struct channel_gk20a *ch,
1240 struct gk20a_cs_snapshot_client *client);
1241 bool (*get_overflow_status)(struct gk20a *g);
1242 u32 (*get_pending_snapshots)(struct gk20a *g);
1243 } css;
1244#endif
1245 struct {
1246 int (*get_speed)(struct gk20a *g, u32 *xve_link_speed);
1247 int (*set_speed)(struct gk20a *g, u32 xve_link_speed);
1248 void (*available_speeds)(struct gk20a *g, u32 *speed_mask);
1249 u32 (*xve_readl)(struct gk20a *g, u32 reg);
1250 void (*xve_writel)(struct gk20a *g, u32 reg, u32 val);
1251 void (*disable_aspm)(struct gk20a *g);
1252 void (*reset_gpu)(struct gk20a *g);
1253#if defined(CONFIG_PCI_MSI)
1254 void (*rearm_msi)(struct gk20a *g);
1255#endif
1256 void (*enable_shadow_rom)(struct gk20a *g);
1257 void (*disable_shadow_rom)(struct gk20a *g);
1258 u32 (*get_link_control_status)(struct gk20a *g);
1259 } xve;
1260 struct {
1261 int (*falcon_hal_sw_init)(struct nvgpu_falcon *flcn);
1262 } falcon;
1263 struct {
1264 void (*enable_priv_ring)(struct gk20a *g);
1265 void (*isr)(struct gk20a *g);
1266 void (*decode_error_code)(struct gk20a *g, u32 error_code);
1267 void (*set_ppriv_timeout_settings)(struct gk20a *g);
1268 u32 (*enum_ltc)(struct gk20a *g);
1269 } priv_ring;
1270 struct {
1271 int (*check_priv_security)(struct gk20a *g);
1272 bool (*is_opt_ecc_enable)(struct gk20a *g);
1273 bool (*is_opt_feature_override_disable)(struct gk20a *g);
1274 u32 (*fuse_status_opt_fbio)(struct gk20a *g);
1275 u32 (*fuse_status_opt_fbp)(struct gk20a *g);
1276 u32 (*fuse_status_opt_rop_l2_fbp)(struct gk20a *g, u32 fbp);
1277 u32 (*fuse_status_opt_tpc_gpc)(struct gk20a *g, u32 gpc);
1278 void (*fuse_ctrl_opt_tpc_gpc)(struct gk20a *g, u32 gpc, u32 val);
1279 u32 (*fuse_opt_sec_debug_en)(struct gk20a *g);
1280 u32 (*fuse_opt_priv_sec_en)(struct gk20a *g);
1281 u32 (*read_vin_cal_fuse_rev)(struct gk20a *g);
1282 u32 (*read_vin_cal_slope_intercept_fuse)(struct gk20a *g,
1283 u32 vin_id, u32 *slope,
1284 u32 *intercept);
1285 u32 (*read_vin_cal_gain_offset_fuse)(struct gk20a *g,
1286 u32 vin_id, s8 *gain,
1287 s8 *offset);
1288 } fuse;
1289 struct {
1290 int (*init)(struct gk20a *g);
1291 int (*discover_ioctrl)(struct gk20a *g);
1292 int (*discover_link)(struct gk20a *g);
1293 int (*isr)(struct gk20a *g);
1294 int (*rxdet)(struct gk20a *g, u32 link_id);
1295 int (*setup_pll)(struct gk20a *g, unsigned long link_mask);
1296 int (*minion_data_ready_en)(struct gk20a *g,
1297 unsigned long link_mask, bool sync);
1298 void (*get_connected_link_mask)(u32 *link_mask);
1299 void (*set_sw_war)(struct gk20a *g, u32 link_id);
1300 /* API */
1301 int (*link_early_init)(struct gk20a *g, unsigned long mask);
1302 u32 (*link_get_mode)(struct gk20a *g, u32 link_id);
1303 u32 (*link_get_state)(struct gk20a *g, u32 link_id);
1304 int (*link_set_mode)(struct gk20a *g, u32 link_id, u32 mode);
1305 u32 (*get_sublink_mode)(struct gk20a *g, u32 link_id,
1306 bool is_rx_sublink);
1307 u32 (*get_rx_sublink_state)(struct gk20a *g, u32 link_id);
1308 u32 (*get_tx_sublink_state)(struct gk20a *g, u32 link_id);
1309 int (*set_sublink_mode)(struct gk20a *g, u32 link_id,
1310 bool is_rx_sublink, u32 mode);
1311 int (*interface_init)(struct gk20a *g);
1312 int (*interface_disable)(struct gk20a *g);
1313 int (*reg_init)(struct gk20a *g);
1314 int (*shutdown)(struct gk20a *g);
1315 int (*early_init)(struct gk20a *g);
1316 } nvlink;
1317 struct {
1318 u32 (*get_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g);
1319 void (*set_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g, u32 val);
1320 u32 (*get_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g);
1321 void (*set_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g, u32 val);
1322 } top;
1323 void (*semaphore_wakeup)(struct gk20a *g, bool post_events);
1324};
1325
1326struct nvgpu_bios_ucode {
1327 u8 *bootloader;
1328 u32 bootloader_phys_base;
1329 u32 bootloader_size;
1330 u8 *ucode;
1331 u32 phys_base;
1332 u32 size;
1333 u8 *dmem;
1334 u32 dmem_phys_base;
1335 u32 dmem_size;
1336 u32 code_entry_point;
1337};
1338
1339struct nvgpu_bios {
1340 u32 vbios_version;
1341 u8 vbios_oem_version;
1342
1343 u8 *data;
1344 size_t size;
1345
1346 struct nvgpu_bios_ucode devinit;
1347 struct nvgpu_bios_ucode preos;
1348
1349 u8 *devinit_tables;
1350 u32 devinit_tables_size;
1351 u8 *bootscripts;
1352 u32 bootscripts_size;
1353
1354 u8 mem_strap_data_count;
1355 u16 mem_strap_xlat_tbl_ptr;
1356
1357 u32 condition_table_ptr;
1358
1359 u32 devinit_tables_phys_base;
1360 u32 devinit_script_phys_base;
1361
1362 struct bit_token *perf_token;
1363 struct bit_token *clock_token;
1364 struct bit_token *virt_token;
1365 u32 expansion_rom_offset;
1366
1367 u32 nvlink_config_data_offset;
1368};
1369
1370struct nvgpu_gpu_params {
1371 /* GPU architecture ID */
1372 u32 gpu_arch;
1373 /* GPU implementation ID */
1374 u32 gpu_impl;
1375 /* GPU revision ID */
1376 u32 gpu_rev;
1377 /* sm version */
1378 u32 sm_arch_sm_version;
1379 /* sm instruction set */
1380 u32 sm_arch_spa_version;
1381 u32 sm_arch_warp_count;
1382};
1383
1384struct gk20a {
1385 void (*free)(struct gk20a *g);
1386 struct nvgpu_nvhost_dev *nvhost_dev;
1387
1388 /*
1389 * Used by <nvgpu/enabled.h>. Do not access directly!
1390 */
1391 unsigned long *enabled_flags;
1392
1393 nvgpu_atomic_t usage_count;
1394
1395 struct nvgpu_mutex ctxsw_disable_lock;
1396 int ctxsw_disable_count;
1397
1398 struct nvgpu_ref refcount;
1399
1400 const char *name;
1401
1402 bool gpu_reset_done;
1403 bool power_on;
1404 bool suspended;
1405 bool sw_ready;
1406
1407 u64 log_mask;
1408 u32 log_trace;
1409
1410 struct nvgpu_mutex tpc_pg_lock;
1411
1412 struct nvgpu_gpu_params params;
1413
1414 /*
1415 * Guards access to hardware when usual gk20a_{busy,idle} are skipped
1416 * for submits and held for channel lifetime but dropped for an ongoing
1417 * gk20a_do_idle().
1418 */
1419 struct nvgpu_rwsem deterministic_busy;
1420
1421 struct nvgpu_falcon pmu_flcn;
1422 struct nvgpu_falcon sec2_flcn;
1423 struct nvgpu_falcon fecs_flcn;
1424 struct nvgpu_falcon gpccs_flcn;
1425 struct nvgpu_falcon nvdec_flcn;
1426 struct nvgpu_falcon minion_flcn;
1427 struct clk_gk20a clk;
1428 struct fifo_gk20a fifo;
1429 struct nvgpu_nvlink_dev nvlink;
1430 struct gr_gk20a gr;
1431 struct sim_nvgpu *sim;
1432 struct mm_gk20a mm;
1433 struct nvgpu_pmu pmu;
1434 struct acr_desc acr;
1435 struct nvgpu_ecc ecc;
1436 struct clk_pmupstate clk_pmu;
1437 struct perf_pmupstate perf_pmu;
1438 struct pmgr_pmupstate pmgr_pmu;
1439 struct therm_pmupstate therm_pmu;
1440
1441#ifdef CONFIG_DEBUG_FS
1442 struct railgate_stats pstats;
1443#endif
1444 u32 gr_idle_timeout_default;
1445 bool timeouts_disabled_by_user;
1446 unsigned int ch_wdt_timeout_ms;
1447 u32 fifo_eng_timeout_us;
1448
1449 struct nvgpu_mutex poweron_lock;
1450 struct nvgpu_mutex poweroff_lock;
1451
1452 /* Channel priorities */
1453 u32 timeslice_low_priority_us;
1454 u32 timeslice_medium_priority_us;
1455 u32 timeslice_high_priority_us;
1456 u32 min_timeslice_us;
1457 u32 max_timeslice_us;
1458 bool runlist_interleave;
1459
1460 bool slcg_enabled;
1461 bool blcg_enabled;
1462 bool elcg_enabled;
1463 bool elpg_enabled;
1464 bool aelpg_enabled;
1465 bool can_elpg;
1466 bool mscg_enabled;
1467 bool forced_idle;
1468 bool forced_reset;
1469 bool allow_all;
1470
1471 u32 ptimer_src_freq;
1472
1473 int railgate_delay;
1474 u8 ldiv_slowdown_factor;
1475 unsigned int aggressive_sync_destroy_thresh;
1476 bool aggressive_sync_destroy;
1477
1478 bool has_syncpoints;
1479 /* Debugfs knob for forcing syncpt support off in runtime. */
1480 u32 disable_syncpoints;
1481
1482 bool support_pmu;
1483 u32 bootstrap_owner;
1484
1485 bool is_virtual;
1486
1487 bool has_cde;
1488
1489 u32 emc3d_ratio;
1490
1491 struct nvgpu_spinlock ltc_enabled_lock;
1492
1493 struct gk20a_ctxsw_ucode_info ctxsw_ucode_info;
1494
1495 /*
1496 * A group of semaphore pools. One for each channel.
1497 */
1498 struct nvgpu_semaphore_sea *sema_sea;
1499
1500 /* held while manipulating # of debug/profiler sessions present */
1501 /* also prevents debug sessions from attaching until released */
1502 struct nvgpu_mutex dbg_sessions_lock;
1503 int dbg_powergating_disabled_refcount; /*refcount for pg disable */
1504 /*refcount for timeout disable */
1505 nvgpu_atomic_t timeouts_disabled_refcount;
1506
1507 /* must have dbg_sessions_lock before use */
1508 struct nvgpu_dbg_reg_op *dbg_regops_tmp_buf;
1509 u32 dbg_regops_tmp_buf_ops;
1510
1511 /* For perfbuf mapping */
1512 struct {
1513 struct dbg_session_gk20a *owner;
1514 u64 offset;
1515 } perfbuf;
1516
1517 /* For profiler reservations */
1518 struct nvgpu_list_node profiler_objects;
1519 bool global_profiler_reservation_held;
1520 int profiler_reservation_count;
1521
1522 void (*remove_support)(struct gk20a *);
1523
1524 u64 pg_ingating_time_us;
1525 u64 pg_ungating_time_us;
1526 u32 pg_gating_cnt;
1527
1528 struct nvgpu_spinlock mc_enable_lock;
1529
1530 struct gk20a_as as;
1531
1532 struct nvgpu_mutex client_lock;
1533 int client_refcount; /* open channels and ctrl nodes */
1534
1535 struct gpu_ops ops;
1536 u32 mc_intr_mask_restore[4];
1537 /*used for change of enum zbc update cmd id from ver 0 to ver1*/
1538 u32 pmu_ver_cmd_id_zbc_table_update;
1539 u32 pmu_lsf_pmu_wpr_init_done;
1540 u32 pmu_lsf_loaded_falcon_id;
1541
1542 int irqs_enabled;
1543 int irq_stall; /* can be same as irq_nonstall in case of PCI */
1544 int irq_nonstall;
1545 u32 max_ltc_count;
1546 u32 ltc_count;
1547 u32 ltc_streamid;
1548
1549 struct gk20a_worker {
1550 struct nvgpu_thread poll_task;
1551 nvgpu_atomic_t put;
1552 struct nvgpu_cond wq;
1553 struct nvgpu_list_node items;
1554 struct nvgpu_spinlock items_lock;
1555 struct nvgpu_mutex start_lock;
1556 } channel_worker, clk_arb_worker;
1557
1558 struct {
1559 void (*open)(struct channel_gk20a *ch);
1560 void (*close)(struct channel_gk20a *ch);
1561 void (*work_completion_signal)(struct channel_gk20a *ch);
1562 void (*work_completion_cancel_sync)(struct channel_gk20a *ch);
1563 bool (*os_fence_framework_inst_exists)(struct channel_gk20a *ch);
1564 int (*init_os_fence_framework)(
1565 struct channel_gk20a *ch, const char *fmt, ...);
1566 void (*signal_os_fence_framework)(struct channel_gk20a *ch);
1567 void (*destroy_os_fence_framework)(struct channel_gk20a *ch);
1568 int (*copy_user_gpfifo)(struct nvgpu_gpfifo_entry *dest,
1569 struct nvgpu_gpfifo_userdata userdata,
1570 u32 start, u32 length);
1571 int (*alloc_usermode_buffers)(struct channel_gk20a *c,
1572 struct nvgpu_gpfifo_args *gpfifo_args);
1573 } os_channel;
1574
1575 struct gk20a_scale_profile *scale_profile;
1576 unsigned long last_freq;
1577
1578 struct gk20a_ctxsw_trace *ctxsw_trace;
1579 struct gk20a_fecs_trace *fecs_trace;
1580
1581 bool mmu_debug_ctrl;
1582
1583 u32 tpc_fs_mask_user;
1584
1585 u32 tpc_pg_mask;
1586 bool can_tpc_powergate;
1587
1588 u32 valid_tpc_mask;
1589
1590 struct nvgpu_bios bios;
1591 bool bios_is_init;
1592
1593 struct nvgpu_clk_arb *clk_arb;
1594
1595 struct nvgpu_mutex clk_arb_enable_lock;
1596
1597 struct gk20a_ce_app ce_app;
1598
1599 bool ltc_intr_en_illegal_compstat;
1600
1601 /* PCI device identifier */
1602 u16 pci_vendor_id, pci_device_id;
1603 u16 pci_subsystem_vendor_id, pci_subsystem_device_id;
1604 u16 pci_class;
1605 u8 pci_revision;
1606
1607 /*
1608 * PCI power management: i2c device index, port and address for
1609 * INA3221.
1610 */
1611 u32 ina3221_dcb_index;
1612 u32 ina3221_i2c_address;
1613 u32 ina3221_i2c_port;
1614 bool hardcode_sw_threshold;
1615
1616 /* PCIe power states. */
1617 bool xve_l0s;
1618 bool xve_l1;
1619
1620 /* Current warning temp in sfxp24.8 */
1621 s32 curr_warn_temp;
1622
1623#if defined(CONFIG_PCI_MSI)
1624 /* Check if msi is enabled */
1625 bool msi_enabled;
1626#endif
1627#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
1628 struct nvgpu_mem_alloc_tracker *vmallocs;
1629 struct nvgpu_mem_alloc_tracker *kmallocs;
1630#endif
1631
1632 /* The minimum VBIOS version supported */
1633 u32 vbios_min_version;
1634
1635 /* memory training sequence and mclk switch scripts */
1636 u32 mem_config_idx;
1637
1638 u64 dma_memory_used;
1639
1640#if defined(CONFIG_TEGRA_GK20A_NVHOST)
1641 u64 syncpt_unit_base;
1642 size_t syncpt_unit_size;
1643 u32 syncpt_size;
1644#endif
1645 struct nvgpu_mem syncpt_mem;
1646
1647 struct nvgpu_list_node boardobj_head;
1648 struct nvgpu_list_node boardobjgrp_head;
1649};
1650
1651static inline bool nvgpu_is_timeouts_enabled(struct gk20a *g)
1652{
1653 return nvgpu_atomic_read(&g->timeouts_disabled_refcount) == 0;
1654}
1655
1656static inline u32 gk20a_get_gr_idle_timeout(struct gk20a *g)
1657{
1658 return nvgpu_is_timeouts_enabled(g) ?
1659 g->gr_idle_timeout_default : UINT_MAX;
1660}
1661
1662#define MULTICHAR_TAG(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d))
1663enum BAR0_DEBUG_OPERATION {
1664 BARO_ZERO_NOP = 0,
1665 OP_END = MULTICHAR_TAG('D', 'O', 'N', 'E'),
1666 BAR0_READ32 = MULTICHAR_TAG('0', 'R', '3', '2'),
1667 BAR0_WRITE32 = MULTICHAR_TAG('0', 'W', '3', '2'),
1668};
1669
1670struct share_buffer_head {
1671 enum BAR0_DEBUG_OPERATION operation;
1672/* size of the operation item */
1673 u32 size;
1674 u32 completed;
1675 u32 failed;
1676 u64 context;
1677 u64 completion_callback;
1678};
1679
1680struct gk20a_cyclestate_buffer_elem {
1681 struct share_buffer_head head;
1682/* in */
1683 u64 p_data;
1684 u64 p_done;
1685 u32 offset_bar0;
1686 u16 first_bit;
1687 u16 last_bit;
1688/* out */
1689/* keep 64 bits to be consistent */
1690 u64 data;
1691};
1692
1693/* operations that will need to be executed on non stall workqueue */
1694#define GK20A_NONSTALL_OPS_WAKEUP_SEMAPHORE BIT32(0)
1695#define GK20A_NONSTALL_OPS_POST_EVENTS BIT32(1)
1696
1697/* register accessors */
1698void __nvgpu_check_gpu_state(struct gk20a *g);
1699void __gk20a_warn_on_no_regs(void);
1700
1701/* classes that the device supports */
1702/* TBD: get these from an open-sourced SDK? */
1703enum {
1704 FERMI_TWOD_A = 0x902D,
1705 KEPLER_INLINE_TO_MEMORY_A = 0xA040,
1706 KEPLER_DMA_COPY_A = 0xA0B5,
1707};
1708
1709#define GK20A_BAR0_IORESOURCE_MEM 0
1710#define GK20A_BAR1_IORESOURCE_MEM 1
1711#define GK20A_SIM_IORESOURCE_MEM 2
1712
1713void gk20a_busy_noresume(struct gk20a *g);
1714void gk20a_idle_nosuspend(struct gk20a *g);
1715int __must_check gk20a_busy(struct gk20a *g);
1716void gk20a_idle(struct gk20a *g);
1717int __gk20a_do_idle(struct gk20a *g, bool force_reset);
1718int __gk20a_do_unidle(struct gk20a *g);
1719
1720int gk20a_can_busy(struct gk20a *g);
1721int gk20a_wait_for_idle(struct gk20a *g);
1722
1723#define NVGPU_GPU_ARCHITECTURE_SHIFT 4
1724
1725/* constructs unique and compact GPUID from nvgpu_gpu_characteristics
1726 * arch/impl fields */
1727#define GK20A_GPUID(arch, impl) ((u32) ((arch) | (impl)))
1728
1729#define GK20A_GPUID_GK20A 0x000000EA
1730#define GK20A_GPUID_GM20B 0x0000012B
1731#define GK20A_GPUID_GM20B_B 0x0000012E
1732#define NVGPU_GPUID_GP10B 0x0000013B
1733#define NVGPU_GPUID_GP104 0x00000134
1734#define NVGPU_GPUID_GP106 0x00000136
1735#define NVGPU_GPUID_GV11B 0x0000015B
1736#define NVGPU_GPUID_GV100 0x00000140
1737
1738int gk20a_init_gpu_characteristics(struct gk20a *g);
1739
1740int gk20a_prepare_poweroff(struct gk20a *g);
1741int gk20a_finalize_poweron(struct gk20a *g);
1742
1743void nvgpu_wait_for_deferred_interrupts(struct gk20a *g);
1744 26
1745struct gk20a * __must_check gk20a_get(struct gk20a *g); 27#ifndef GK20A_GK20A_H
1746void gk20a_put(struct gk20a *g); 28#define GK20A_GK20A_H
1747 29
1748static inline bool gk20a_platform_has_syncpoints(struct gk20a *g) 30/* no new headers should be added here */
1749{ 31#include <nvgpu/gk20a.h>
1750#ifdef CONFIG_TEGRA_GK20A_NVHOST
1751 return g->has_syncpoints && !g->disable_syncpoints;
1752#else
1753 return false;
1754#endif
1755}
1756 32
1757int gk20a_detect_chip(struct gk20a *g); 33#endif \ No newline at end of file
1758#endif /* GK20A_H */
diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h
new file mode 100644
index 00000000..31ca1b45
--- /dev/null
+++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h
@@ -0,0 +1,1758 @@
1/*
2 * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * GK20A Graphics
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24#ifndef GK20A_H
25#define GK20A_H
26
27struct gk20a;
28struct fifo_gk20a;
29struct channel_gk20a;
30struct gr_gk20a;
31struct sim_nvgpu;
32struct gk20a_ctxsw_ucode_segments;
33struct gk20a_fecs_trace;
34struct gk20a_ctxsw_trace;
35struct acr_desc;
36struct nvgpu_mem_alloc_tracker;
37struct dbg_profiler_object_data;
38struct gk20a_debug_output;
39struct nvgpu_clk_pll_debug_data;
40struct nvgpu_nvhost_dev;
41struct nvgpu_cpu_time_correlation_sample;
42struct nvgpu_mem_sgt;
43struct nvgpu_warpstate;
44struct nvgpu_clk_session;
45struct nvgpu_clk_arb;
46#ifdef CONFIG_GK20A_CTXSW_TRACE
47struct nvgpu_gpu_ctxsw_trace_filter;
48#endif
49struct priv_cmd_entry;
50struct nvgpu_gpfifo_args;
51
52#include <nvgpu/lock.h>
53#include <nvgpu/thread.h>
54
55#include <nvgpu/mm.h>
56#include <nvgpu/as.h>
57#include <nvgpu/log.h>
58#include <nvgpu/pramin.h>
59#include <nvgpu/acr/nvgpu_acr.h>
60#include <nvgpu/kref.h>
61#include <nvgpu/falcon.h>
62#include <nvgpu/pmu.h>
63#include <nvgpu/atomic.h>
64#include <nvgpu/barrier.h>
65#include <nvgpu/rwsem.h>
66#include <nvgpu/nvlink.h>
67#include <nvgpu/sim.h>
68#include <nvgpu/ecc.h>
69
70#include "gk20a/clk_gk20a.h"
71#include "gk20a/ce2_gk20a.h"
72#include "gk20a/fifo_gk20a.h"
73#include "gk20a/tsg_gk20a.h"
74#include "clk/clk.h"
75#include "perf/perf.h"
76#include "pmgr/pmgr.h"
77#include "therm/thrm.h"
78
79#ifdef CONFIG_DEBUG_FS
80struct railgate_stats {
81 unsigned long last_rail_gate_start;
82 unsigned long last_rail_gate_complete;
83 unsigned long last_rail_ungate_start;
84 unsigned long last_rail_ungate_complete;
85 unsigned long total_rail_gate_time_ms;
86 unsigned long total_rail_ungate_time_ms;
87 unsigned long railgating_cycle_count;
88};
89#endif
90
91enum gk20a_cbc_op {
92 gk20a_cbc_op_clear,
93 gk20a_cbc_op_clean,
94 gk20a_cbc_op_invalidate,
95};
96
97#define MC_INTR_UNIT_DISABLE false
98#define MC_INTR_UNIT_ENABLE true
99
100#define GPU_LIT_NUM_GPCS 0
101#define GPU_LIT_NUM_PES_PER_GPC 1
102#define GPU_LIT_NUM_ZCULL_BANKS 2
103#define GPU_LIT_NUM_TPC_PER_GPC 3
104#define GPU_LIT_NUM_SM_PER_TPC 4
105#define GPU_LIT_NUM_FBPS 5
106#define GPU_LIT_GPC_BASE 6
107#define GPU_LIT_GPC_STRIDE 7
108#define GPU_LIT_GPC_SHARED_BASE 8
109#define GPU_LIT_TPC_IN_GPC_BASE 9
110#define GPU_LIT_TPC_IN_GPC_STRIDE 10
111#define GPU_LIT_TPC_IN_GPC_SHARED_BASE 11
112#define GPU_LIT_PPC_IN_GPC_BASE 12
113#define GPU_LIT_PPC_IN_GPC_STRIDE 13
114#define GPU_LIT_PPC_IN_GPC_SHARED_BASE 14
115#define GPU_LIT_ROP_BASE 15
116#define GPU_LIT_ROP_STRIDE 16
117#define GPU_LIT_ROP_SHARED_BASE 17
118#define GPU_LIT_HOST_NUM_ENGINES 18
119#define GPU_LIT_HOST_NUM_PBDMA 19
120#define GPU_LIT_LTC_STRIDE 20
121#define GPU_LIT_LTS_STRIDE 21
122#define GPU_LIT_NUM_FBPAS 22
123#define GPU_LIT_FBPA_STRIDE 23
124#define GPU_LIT_FBPA_BASE 24
125#define GPU_LIT_FBPA_SHARED_BASE 25
126#define GPU_LIT_SM_PRI_STRIDE 26
127#define GPU_LIT_SMPC_PRI_BASE 27
128#define GPU_LIT_SMPC_PRI_SHARED_BASE 28
129#define GPU_LIT_SMPC_PRI_UNIQUE_BASE 29
130#define GPU_LIT_SMPC_PRI_STRIDE 30
131#define GPU_LIT_TWOD_CLASS 31
132#define GPU_LIT_THREED_CLASS 32
133#define GPU_LIT_COMPUTE_CLASS 33
134#define GPU_LIT_GPFIFO_CLASS 34
135#define GPU_LIT_I2M_CLASS 35
136#define GPU_LIT_DMA_COPY_CLASS 36
137#define GPU_LIT_GPC_PRIV_STRIDE 37
138
139#define nvgpu_get_litter_value(g, v) (g)->ops.get_litter_value((g), v)
140
141enum nvgpu_unit;
142
143enum nvgpu_flush_op;
144enum gk20a_mem_rw_flag;
145
146struct _resmgr_context;
147struct nvgpu_gpfifo_entry;
148
149struct nvgpu_gpfifo_userdata {
150 struct nvgpu_gpfifo_entry __user *entries;
151 struct _resmgr_context *context;
152};
153
154/*
155 * gpu_ops should only contain function pointers! Non-function pointer members
156 * should go in struct gk20a or be implemented with the boolean flag API defined
157 * in nvgpu/enabled.h
158 */
159
160/* index for FB fault buffer functions */
161#define NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX 0U
162#define NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX 1U
163#define NVGPU_FB_MMU_FAULT_BUF_DISABLED 0U
164#define NVGPU_FB_MMU_FAULT_BUF_ENABLED 1U
165
166/* Parameters for init_elcg_mode/init_blcg_mode */
167enum {
168 ELCG_RUN, /* clk always run, i.e. disable elcg */
169 ELCG_STOP, /* clk is stopped */
170 ELCG_AUTO /* clk will run when non-idle, standard elcg mode */
171};
172
173enum {
174 BLCG_RUN, /* clk always run, i.e. disable blcg */
175 BLCG_AUTO /* clk will run when non-idle, standard blcg mode */
176};
177
178struct gpu_ops {
179 struct {
180 int (*determine_L2_size_bytes)(struct gk20a *gk20a);
181 u64 (*get_cbc_base_divisor)(struct gk20a *g);
182 int (*init_comptags)(struct gk20a *g, struct gr_gk20a *gr);
183 int (*cbc_ctrl)(struct gk20a *g, enum gk20a_cbc_op op,
184 u32 min, u32 max);
185 void (*set_zbc_color_entry)(struct gk20a *g,
186 struct zbc_entry *color_val,
187 u32 index);
188 void (*set_zbc_depth_entry)(struct gk20a *g,
189 struct zbc_entry *depth_val,
190 u32 index);
191 void (*set_zbc_s_entry)(struct gk20a *g,
192 struct zbc_entry *s_val,
193 u32 index);
194 void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr);
195 void (*set_enabled)(struct gk20a *g, bool enabled);
196 void (*init_fs_state)(struct gk20a *g);
197 void (*isr)(struct gk20a *g);
198 u32 (*cbc_fix_config)(struct gk20a *g, int base);
199 void (*flush)(struct gk20a *g);
200 void (*intr_en_illegal_compstat)(struct gk20a *g, bool enable);
201 bool (*pri_is_ltc_addr)(struct gk20a *g, u32 addr);
202 bool (*is_ltcs_ltss_addr)(struct gk20a *g, u32 addr);
203 bool (*is_ltcn_ltss_addr)(struct gk20a *g, u32 addr);
204 void (*split_lts_broadcast_addr)(struct gk20a *g, u32 addr,
205 u32 *priv_addr_table,
206 u32 *priv_addr_table_index);
207 void (*split_ltc_broadcast_addr)(struct gk20a *g, u32 addr,
208 u32 *priv_addr_table,
209 u32 *priv_addr_table_index);
210 } ltc;
211 struct {
212 void (*isr_stall)(struct gk20a *g, u32 inst_id, u32 pri_base);
213 u32 (*isr_nonstall)(struct gk20a *g, u32 inst_id, u32 pri_base);
214 u32 (*get_num_pce)(struct gk20a *g);
215 } ce2;
216 struct {
217 u32 (*get_patch_slots)(struct gk20a *g);
218 int (*init_fs_state)(struct gk20a *g);
219 int (*init_preemption_state)(struct gk20a *g);
220 void (*access_smpc_reg)(struct gk20a *g, u32 quad, u32 offset);
221 void (*bundle_cb_defaults)(struct gk20a *g);
222 void (*cb_size_default)(struct gk20a *g);
223 int (*calc_global_ctx_buffer_size)(struct gk20a *g);
224 void (*commit_global_attrib_cb)(struct gk20a *g,
225 struct nvgpu_gr_ctx *ch_ctx,
226 u64 addr, bool patch);
227 void (*commit_global_bundle_cb)(struct gk20a *g,
228 struct nvgpu_gr_ctx *ch_ctx,
229 u64 addr, u64 size, bool patch);
230 int (*commit_global_cb_manager)(struct gk20a *g,
231 struct channel_gk20a *ch,
232 bool patch);
233 void (*commit_global_pagepool)(struct gk20a *g,
234 struct nvgpu_gr_ctx *ch_ctx,
235 u64 addr, u32 size, bool patch);
236 void (*init_gpc_mmu)(struct gk20a *g);
237 int (*handle_sw_method)(struct gk20a *g, u32 addr,
238 u32 class_num, u32 offset, u32 data);
239 void (*set_alpha_circular_buffer_size)(struct gk20a *g,
240 u32 data);
241 void (*set_circular_buffer_size)(struct gk20a *g, u32 data);
242 void (*set_bes_crop_debug3)(struct gk20a *g, u32 data);
243 void (*set_bes_crop_debug4)(struct gk20a *g, u32 data);
244 void (*enable_hww_exceptions)(struct gk20a *g);
245 bool (*is_valid_class)(struct gk20a *g, u32 class_num);
246 bool (*is_valid_gfx_class)(struct gk20a *g, u32 class_num);
247 bool (*is_valid_compute_class)(struct gk20a *g, u32 class_num);
248 void (*get_sm_dsm_perf_regs)(struct gk20a *g,
249 u32 *num_sm_dsm_perf_regs,
250 u32 **sm_dsm_perf_regs,
251 u32 *perf_register_stride);
252 void (*get_sm_dsm_perf_ctrl_regs)(struct gk20a *g,
253 u32 *num_sm_dsm_perf_regs,
254 u32 **sm_dsm_perf_regs,
255 u32 *perf_register_stride);
256 void (*get_ovr_perf_regs)(struct gk20a *g,
257 u32 *num_ovr_perf_regs,
258 u32 **ovr_perf_regsr);
259 void (*set_hww_esr_report_mask)(struct gk20a *g);
260 int (*setup_alpha_beta_tables)(struct gk20a *g,
261 struct gr_gk20a *gr);
262 int (*falcon_load_ucode)(struct gk20a *g,
263 u64 addr_base,
264 struct gk20a_ctxsw_ucode_segments *segments,
265 u32 reg_offset);
266 int (*load_ctxsw_ucode)(struct gk20a *g);
267 u32 (*get_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
268 void (*set_gpc_tpc_mask)(struct gk20a *g, u32 gpc_index);
269 int (*alloc_obj_ctx)(struct channel_gk20a *c,
270 u32 class_num, u32 flags);
271 int (*bind_ctxsw_zcull)(struct gk20a *g, struct gr_gk20a *gr,
272 struct channel_gk20a *c, u64 zcull_va,
273 u32 mode);
274 int (*get_zcull_info)(struct gk20a *g, struct gr_gk20a *gr,
275 struct gr_zcull_info *zcull_params);
276 int (*decode_egpc_addr)(struct gk20a *g,
277 u32 addr, enum ctxsw_addr_type *addr_type,
278 u32 *gpc_num, u32 *tpc_num, u32 *broadcast_flags);
279 void (*egpc_etpc_priv_addr_table)(struct gk20a *g, u32 addr,
280 u32 gpc, u32 tpc, u32 broadcast_flags,
281 u32 *priv_addr_table,
282 u32 *priv_addr_table_index);
283 bool (*is_tpc_addr)(struct gk20a *g, u32 addr);
284 bool (*is_egpc_addr)(struct gk20a *g, u32 addr);
285 bool (*is_etpc_addr)(struct gk20a *g, u32 addr);
286 void (*get_egpc_etpc_num)(struct gk20a *g, u32 addr,
287 u32 *gpc_num, u32 *tpc_num);
288 u32 (*get_tpc_num)(struct gk20a *g, u32 addr);
289 u32 (*get_egpc_base)(struct gk20a *g);
290 void (*detect_sm_arch)(struct gk20a *g);
291 int (*add_zbc_color)(struct gk20a *g, struct gr_gk20a *gr,
292 struct zbc_entry *color_val, u32 index);
293 int (*add_zbc_depth)(struct gk20a *g, struct gr_gk20a *gr,
294 struct zbc_entry *depth_val, u32 index);
295 int (*add_zbc_s)(struct gk20a *g, struct gr_gk20a *gr,
296 struct zbc_entry *s_val, u32 index);
297 int (*zbc_set_table)(struct gk20a *g, struct gr_gk20a *gr,
298 struct zbc_entry *zbc_val);
299 int (*zbc_query_table)(struct gk20a *g, struct gr_gk20a *gr,
300 struct zbc_query_params *query_params);
301 int (*zbc_s_query_table)(struct gk20a *g, struct gr_gk20a *gr,
302 struct zbc_query_params *query_params);
303 int (*load_zbc_s_default_tbl)(struct gk20a *g,
304 struct gr_gk20a *gr);
305 int (*load_zbc_s_tbl)(struct gk20a *g,
306 struct gr_gk20a *gr);
307 void (*pmu_save_zbc)(struct gk20a *g, u32 entries);
308 int (*add_zbc)(struct gk20a *g, struct gr_gk20a *gr,
309 struct zbc_entry *zbc_val);
310 bool (*add_zbc_type_s)(struct gk20a *g, struct gr_gk20a *gr,
311 struct zbc_entry *zbc_val, int *ret_val);
312 u32 (*pagepool_default_size)(struct gk20a *g);
313 int (*init_ctx_state)(struct gk20a *g);
314 int (*alloc_gr_ctx)(struct gk20a *g,
315 struct nvgpu_gr_ctx *gr_ctx, struct vm_gk20a *vm,
316 u32 class, u32 padding);
317 void (*free_gr_ctx)(struct gk20a *g,
318 struct vm_gk20a *vm, struct nvgpu_gr_ctx *gr_ctx);
319 void (*powergate_tpc)(struct gk20a *g);
320 void (*update_ctxsw_preemption_mode)(struct gk20a *g,
321 struct channel_gk20a *c,
322 struct nvgpu_mem *mem);
323 int (*update_smpc_ctxsw_mode)(struct gk20a *g,
324 struct channel_gk20a *c,
325 bool enable);
326 u32 (*get_hw_accessor_stream_out_mode)(void);
327 int (*update_hwpm_ctxsw_mode)(struct gk20a *g,
328 struct channel_gk20a *c,
329 u64 gpu_va,
330 u32 mode);
331 void (*init_hwpm_pmm_register)(struct gk20a *g);
332 int (*dump_gr_regs)(struct gk20a *g,
333 struct gk20a_debug_output *o);
334 int (*update_pc_sampling)(struct channel_gk20a *ch,
335 bool enable);
336 u32 (*get_max_fbps_count)(struct gk20a *g);
337 u32 (*get_fbp_en_mask)(struct gk20a *g);
338 u32 (*get_max_ltc_per_fbp)(struct gk20a *g);
339 u32 (*get_max_lts_per_ltc)(struct gk20a *g);
340 u32* (*get_rop_l2_en_mask)(struct gk20a *g);
341 void (*init_sm_dsm_reg_info)(void);
342 void (*init_ovr_sm_dsm_perf)(void);
343 int (*wait_empty)(struct gk20a *g, unsigned long duration_ms,
344 u32 expect_delay);
345 void (*init_cyclestats)(struct gk20a *g);
346 void (*enable_cde_in_fecs)(struct gk20a *g,
347 struct nvgpu_mem *mem);
348 int (*set_sm_debug_mode)(struct gk20a *g, struct channel_gk20a *ch,
349 u64 sms, bool enable);
350 void (*bpt_reg_info)(struct gk20a *g,
351 struct nvgpu_warpstate *w_state);
352 void (*get_access_map)(struct gk20a *g,
353 u32 **whitelist, int *num_entries);
354 int (*handle_fecs_error)(struct gk20a *g,
355 struct channel_gk20a *ch,
356 struct gr_gk20a_isr_data *isr_data);
357 int (*pre_process_sm_exception)(struct gk20a *g,
358 u32 gpc, u32 tpc, u32 sm, u32 global_esr, u32 warp_esr,
359 bool sm_debugger_attached,
360 struct channel_gk20a *fault_ch,
361 bool *early_exit, bool *ignore_debugger);
362 u32 (*get_sm_hww_warp_esr)(struct gk20a *g,
363 u32 gpc, u32 tpc, u32 sm);
364 u32 (*get_sm_hww_global_esr)(struct gk20a *g,
365 u32 gpc, u32 tpc, u32 sm);
366 u32 (*get_sm_no_lock_down_hww_global_esr_mask)(struct gk20a *g);
367 int (*lock_down_sm)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm,
368 u32 global_esr_mask, bool check_errors);
369 int (*wait_for_sm_lock_down)(struct gk20a *g, u32 gpc, u32 tpc,
370 u32 sm, u32 global_esr_mask, bool check_errors);
371 void (*clear_sm_hww)(struct gk20a *g, u32 gpc, u32 tpc, u32 sm,
372 u32 global_esr);
373 void (*get_esr_sm_sel)(struct gk20a *g, u32 gpc, u32 tpc,
374 u32 *esr_sm_sel);
375 int (*handle_tpc_sm_ecc_exception)(struct gk20a *g,
376 u32 gpc, u32 tpc,
377 bool *post_event, struct channel_gk20a *fault_ch,
378 u32 *hww_global_esr);
379 int (*handle_sm_exception)(struct gk20a *g,
380 u32 gpc, u32 tpc, u32 sm,
381 bool *post_event, struct channel_gk20a *fault_ch,
382 u32 *hww_global_esr);
383 int (*handle_gcc_exception)(struct gk20a *g, u32 gpc, u32 tpc,
384 bool *post_event, struct channel_gk20a *fault_ch,
385 u32 *hww_global_esr);
386 int (*handle_tex_exception)(struct gk20a *g, u32 gpc, u32 tpc,
387 bool *post_event);
388 int (*handle_tpc_mpc_exception)(struct gk20a *g,
389 u32 gpc, u32 tpc, bool *post_event);
390 int (*handle_gpc_gpccs_exception)(struct gk20a *g, u32 gpc,
391 u32 gpc_exception);
392 int (*handle_gpc_gpcmmu_exception)(struct gk20a *g, u32 gpc,
393 u32 gpc_exception);
394 void (*enable_gpc_exceptions)(struct gk20a *g);
395 void (*enable_exceptions)(struct gk20a *g);
396 int (*init_ecc)(struct gk20a *g);
397 u32 (*get_lrf_tex_ltc_dram_override)(struct gk20a *g);
398 int (*record_sm_error_state)(struct gk20a *g, u32 gpc, u32 tpc,
399 u32 sm, struct channel_gk20a *fault_ch);
400 int (*update_sm_error_state)(struct gk20a *g,
401 struct channel_gk20a *ch, u32 sm_id,
402 struct nvgpu_tsg_sm_error_state *sm_error_state);
403 int (*clear_sm_error_state)(struct gk20a *g,
404 struct channel_gk20a *ch, u32 sm_id);
405 int (*suspend_contexts)(struct gk20a *g,
406 struct dbg_session_gk20a *dbg_s,
407 int *ctx_resident_ch_fd);
408 int (*resume_contexts)(struct gk20a *g,
409 struct dbg_session_gk20a *dbg_s,
410 int *ctx_resident_ch_fd);
411 int (*set_preemption_mode)(struct channel_gk20a *ch,
412 u32 graphics_preempt_mode,
413 u32 compute_preempt_mode);
414 int (*get_preemption_mode_flags)(struct gk20a *g,
415 struct nvgpu_preemption_modes_rec *preemption_modes_rec);
416 int (*set_ctxsw_preemption_mode)(struct gk20a *g,
417 struct nvgpu_gr_ctx *gr_ctx,
418 struct vm_gk20a *vm, u32 class,
419 u32 graphics_preempt_mode,
420 u32 compute_preempt_mode);
421 int (*set_boosted_ctx)(struct channel_gk20a *ch, bool boost);
422 void (*update_boosted_ctx)(struct gk20a *g,
423 struct nvgpu_mem *mem,
424 struct nvgpu_gr_ctx *gr_ctx);
425 int (*init_sm_id_table)(struct gk20a *g);
426 int (*load_smid_config)(struct gk20a *g);
427 void (*program_sm_id_numbering)(struct gk20a *g,
428 u32 gpc, u32 tpc, u32 smid);
429 void (*program_active_tpc_counts)(struct gk20a *g, u32 gpc);
430 int (*setup_rop_mapping)(struct gk20a *g, struct gr_gk20a *gr);
431 int (*init_sw_veid_bundle)(struct gk20a *g);
432 void (*program_zcull_mapping)(struct gk20a *g,
433 u32 zcull_alloc_num, u32 *zcull_map_tiles);
434 int (*commit_global_timeslice)(struct gk20a *g,
435 struct channel_gk20a *c);
436 int (*commit_inst)(struct channel_gk20a *c, u64 gpu_va);
437 void (*write_zcull_ptr)(struct gk20a *g,
438 struct nvgpu_mem *mem, u64 gpu_va);
439 void (*write_pm_ptr)(struct gk20a *g,
440 struct nvgpu_mem *mem, u64 gpu_va);
441 void (*set_preemption_buffer_va)(struct gk20a *g,
442 struct nvgpu_mem *mem, u64 gpu_va);
443 void (*load_tpc_mask)(struct gk20a *g);
444 int (*trigger_suspend)(struct gk20a *g);
445 int (*wait_for_pause)(struct gk20a *g, struct nvgpu_warpstate *w_state);
446 int (*resume_from_pause)(struct gk20a *g);
447 int (*clear_sm_errors)(struct gk20a *g);
448 u32 (*tpc_enabled_exceptions)(struct gk20a *g);
449 int (*set_czf_bypass)(struct gk20a *g,
450 struct channel_gk20a *ch);
451 void (*init_czf_bypass)(struct gk20a *g);
452 bool (*sm_debugger_attached)(struct gk20a *g);
453 void (*suspend_single_sm)(struct gk20a *g,
454 u32 gpc, u32 tpc, u32 sm,
455 u32 global_esr_mask, bool check_errors);
456 void (*suspend_all_sms)(struct gk20a *g,
457 u32 global_esr_mask, bool check_errors);
458 void (*resume_single_sm)(struct gk20a *g,
459 u32 gpc, u32 tpc, u32 sm);
460 void (*resume_all_sms)(struct gk20a *g);
461 void (*disable_rd_coalesce)(struct gk20a *g);
462 void (*init_ctxsw_hdr_data)(struct gk20a *g,
463 struct nvgpu_mem *mem);
464 void (*init_gfxp_wfi_timeout_count)(struct gk20a *g);
465 unsigned long (*get_max_gfxp_wfi_timeout_count)
466 (struct gk20a *g);
467 void (*ecc_init_scrub_reg)(struct gk20a *g);
468 u32 (*get_gpcs_swdx_dss_zbc_c_format_reg)(struct gk20a *g);
469 u32 (*get_gpcs_swdx_dss_zbc_z_format_reg)(struct gk20a *g);
470 void (*dump_ctxsw_stats)(struct gk20a *g, struct vm_gk20a *vm,
471 struct nvgpu_gr_ctx *gr_ctx);
472 void (*fecs_host_int_enable)(struct gk20a *g);
473 int (*handle_ssync_hww)(struct gk20a *g);
474 int (*handle_notify_pending)(struct gk20a *g,
475 struct gr_gk20a_isr_data *isr_data);
476 int (*handle_semaphore_pending)(struct gk20a *g,
477 struct gr_gk20a_isr_data *isr_data);
478 int (*add_ctxsw_reg_pm_fbpa)(struct gk20a *g,
479 struct ctxsw_buf_offset_map_entry *map,
480 struct aiv_list_gk20a *regs,
481 u32 *count, u32 *offset,
482 u32 max_cnt, u32 base,
483 u32 num_fbpas, u32 stride, u32 mask);
484 int (*add_ctxsw_reg_perf_pma)(struct ctxsw_buf_offset_map_entry *map,
485 struct aiv_list_gk20a *regs,
486 u32 *count, u32 *offset,
487 u32 max_cnt, u32 base, u32 mask);
488 int (*decode_priv_addr)(struct gk20a *g, u32 addr,
489 enum ctxsw_addr_type *addr_type,
490 u32 *gpc_num, u32 *tpc_num,
491 u32 *ppc_num, u32 *be_num,
492 u32 *broadcast_flags);
493 int (*create_priv_addr_table)(struct gk20a *g,
494 u32 addr,
495 u32 *priv_addr_table,
496 u32 *num_registers);
497 u32 (*get_pmm_per_chiplet_offset)(void);
498 void (*split_fbpa_broadcast_addr)(struct gk20a *g, u32 addr,
499 u32 num_fbpas,
500 u32 *priv_addr_table,
501 u32 *priv_addr_table_index);
502 u32 (*fecs_ctxsw_mailbox_size)(void);
503 int (*init_sw_bundle64)(struct gk20a *g);
504 int (*alloc_global_ctx_buffers)(struct gk20a *g);
505 int (*map_global_ctx_buffers)(struct gk20a *g,
506 struct channel_gk20a *c);
507 int (*commit_global_ctx_buffers)(struct gk20a *g,
508 struct channel_gk20a *c, bool patch);
509 u32 (*get_nonpes_aware_tpc)(struct gk20a *g, u32 gpc, u32 tpc);
510 int (*get_offset_in_gpccs_segment)(struct gk20a *g,
511 enum ctxsw_addr_type addr_type, u32 num_tpcs,
512 u32 num_ppcs, u32 reg_list_ppc_count,
513 u32 *__offset_in_segment);
514 void (*set_debug_mode)(struct gk20a *g, bool enable);
515 } gr;
516 struct {
517 void (*init_hw)(struct gk20a *g);
518 void (*init_cbc)(struct gk20a *g, struct gr_gk20a *gr);
519 void (*init_fs_state)(struct gk20a *g);
520 void (*reset)(struct gk20a *g);
521 void (*init_uncompressed_kind_map)(struct gk20a *g);
522 void (*init_kind_attr)(struct gk20a *g);
523 void (*set_mmu_page_size)(struct gk20a *g);
524 bool (*set_use_full_comp_tag_line)(struct gk20a *g);
525 u32 (*mmu_ctrl)(struct gk20a *g);
526 u32 (*mmu_debug_ctrl)(struct gk20a *g);
527 u32 (*mmu_debug_wr)(struct gk20a *g);
528 u32 (*mmu_debug_rd)(struct gk20a *g);
529
530 /*
531 * Compression tag line coverage. When mapping a compressible
532 * buffer, ctagline is increased when the virtual address
533 * crosses over the compression page boundary.
534 */
535 unsigned int (*compression_page_size)(struct gk20a *g);
536
537 /*
538 * Minimum page size that can be used for compressible kinds.
539 */
540 unsigned int (*compressible_page_size)(struct gk20a *g);
541
542 /*
543 * Compressible kind mappings: Mask for the virtual and physical
544 * address bits that must match.
545 */
546 u32 (*compression_align_mask)(struct gk20a *g);
547
548 void (*dump_vpr_wpr_info)(struct gk20a *g);
549 int (*vpr_info_fetch)(struct gk20a *g);
550 void (*read_wpr_info)(struct gk20a *g,
551 struct wpr_carveout_info *inf);
552 bool (*is_debug_mode_enabled)(struct gk20a *g);
553 void (*set_debug_mode)(struct gk20a *g, bool enable);
554 int (*tlb_invalidate)(struct gk20a *g, struct nvgpu_mem *pdb);
555 void (*hub_isr)(struct gk20a *g);
556 void (*handle_replayable_fault)(struct gk20a *g);
557 int (*mem_unlock)(struct gk20a *g);
558 int (*init_nvlink)(struct gk20a *g);
559 int (*enable_nvlink)(struct gk20a *g);
560 void (*enable_hub_intr)(struct gk20a *g);
561 void (*disable_hub_intr)(struct gk20a *g);
562 int (*init_fbpa)(struct gk20a *g);
563 void (*fbpa_isr)(struct gk20a *g);
564 void (*write_mmu_fault_buffer_lo_hi)(struct gk20a *g, u32 index,
565 u32 addr_lo, u32 addr_hi);
566 void (*write_mmu_fault_buffer_get)(struct gk20a *g, u32 index,
567 u32 reg_val);
568 void (*write_mmu_fault_buffer_size)(struct gk20a *g, u32 index,
569 u32 reg_val);
570 void (*write_mmu_fault_status)(struct gk20a *g, u32 reg_val);
571 u32 (*read_mmu_fault_buffer_get)(struct gk20a *g, u32 index);
572 u32 (*read_mmu_fault_buffer_put)(struct gk20a *g, u32 index);
573 u32 (*read_mmu_fault_buffer_size)(struct gk20a *g, u32 index);
574 void (*read_mmu_fault_addr_lo_hi)(struct gk20a *g,
575 u32 *addr_lo, u32 *addr_hi);
576 void (*read_mmu_fault_inst_lo_hi)(struct gk20a *g,
577 u32 *inst_lo, u32 *inst_hi);
578 u32 (*read_mmu_fault_info)(struct gk20a *g);
579 u32 (*read_mmu_fault_status)(struct gk20a *g);
580 int (*mmu_invalidate_replay)(struct gk20a *g,
581 u32 invalidate_replay_val);
582 bool (*mmu_fault_pending)(struct gk20a *g);
583 bool (*is_fault_buf_enabled)(struct gk20a *g, u32 index);
584 void (*fault_buf_set_state_hw)(struct gk20a *g,
585 u32 index, u32 state);
586 void (*fault_buf_configure_hw)(struct gk20a *g, u32 index);
587 size_t (*get_vidmem_size)(struct gk20a *g);
588 } fb;
589 struct {
590 void (*slcg_bus_load_gating_prod)(struct gk20a *g, bool prod);
591 void (*slcg_ce2_load_gating_prod)(struct gk20a *g, bool prod);
592 void (*slcg_chiplet_load_gating_prod)(struct gk20a *g, bool prod);
593 void (*slcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod);
594 void (*slcg_fb_load_gating_prod)(struct gk20a *g, bool prod);
595 void (*slcg_fifo_load_gating_prod)(struct gk20a *g, bool prod);
596 void (*slcg_gr_load_gating_prod)(struct gk20a *g, bool prod);
597 void (*slcg_ltc_load_gating_prod)(struct gk20a *g, bool prod);
598 void (*slcg_perf_load_gating_prod)(struct gk20a *g, bool prod);
599 void (*slcg_priring_load_gating_prod)(struct gk20a *g, bool prod);
600 void (*slcg_pmu_load_gating_prod)(struct gk20a *g, bool prod);
601 void (*slcg_therm_load_gating_prod)(struct gk20a *g, bool prod);
602 void (*slcg_xbar_load_gating_prod)(struct gk20a *g, bool prod);
603 void (*blcg_bus_load_gating_prod)(struct gk20a *g, bool prod);
604 void (*blcg_ce_load_gating_prod)(struct gk20a *g, bool prod);
605 void (*blcg_ctxsw_firmware_load_gating_prod)(struct gk20a *g, bool prod);
606 void (*blcg_fb_load_gating_prod)(struct gk20a *g, bool prod);
607 void (*blcg_fifo_load_gating_prod)(struct gk20a *g, bool prod);
608 void (*blcg_gr_load_gating_prod)(struct gk20a *g, bool prod);
609 void (*blcg_ltc_load_gating_prod)(struct gk20a *g, bool prod);
610 void (*blcg_pwr_csb_load_gating_prod)(struct gk20a *g, bool prod);
611 void (*blcg_pmu_load_gating_prod)(struct gk20a *g, bool prod);
612 void (*blcg_xbar_load_gating_prod)(struct gk20a *g, bool prod);
613 void (*pg_gr_load_gating_prod)(struct gk20a *g, bool prod);
614 } clock_gating;
615 struct {
616 void (*post_events)(struct channel_gk20a *ch);
617 } debugger;
618 struct {
619 int (*setup_sw)(struct gk20a *g);
620 int (*init_fifo_setup_hw)(struct gk20a *g);
621 void (*bind_channel)(struct channel_gk20a *ch_gk20a);
622 void (*unbind_channel)(struct channel_gk20a *ch_gk20a);
623 void (*disable_channel)(struct channel_gk20a *ch);
624 void (*enable_channel)(struct channel_gk20a *ch);
625 int (*alloc_inst)(struct gk20a *g, struct channel_gk20a *ch);
626 void (*free_inst)(struct gk20a *g, struct channel_gk20a *ch);
627 int (*setup_ramfc)(struct channel_gk20a *c, u64 gpfifo_base,
628 u32 gpfifo_entries,
629 unsigned long acquire_timeout,
630 u32 flags);
631 int (*resetup_ramfc)(struct channel_gk20a *c);
632 int (*preempt_channel)(struct gk20a *g, u32 chid);
633 int (*preempt_tsg)(struct gk20a *g, u32 tsgid);
634 int (*enable_tsg)(struct tsg_gk20a *tsg);
635 int (*disable_tsg)(struct tsg_gk20a *tsg);
636 int (*tsg_verify_channel_status)(struct channel_gk20a *ch);
637 void (*tsg_verify_status_ctx_reload)(struct channel_gk20a *ch);
638 void (*tsg_verify_status_faulted)(struct channel_gk20a *ch);
639 int (*reschedule_runlist)(struct channel_gk20a *ch,
640 bool preempt_next);
641 int (*update_runlist)(struct gk20a *g, u32 runlist_id,
642 u32 chid, bool add,
643 bool wait_for_finish);
644 void (*trigger_mmu_fault)(struct gk20a *g,
645 unsigned long engine_ids);
646 void (*get_mmu_fault_info)(struct gk20a *g, u32 mmu_fault_id,
647 struct mmu_fault_info *mmfault);
648 void (*get_mmu_fault_desc)(struct mmu_fault_info *mmfault);
649 void (*get_mmu_fault_client_desc)(
650 struct mmu_fault_info *mmfault);
651 void (*get_mmu_fault_gpc_desc)(struct mmu_fault_info *mmfault);
652 void (*apply_pb_timeout)(struct gk20a *g);
653 void (*apply_ctxsw_timeout_intr)(struct gk20a *g);
654 int (*wait_engine_idle)(struct gk20a *g);
655 u32 (*get_num_fifos)(struct gk20a *g);
656 u32 (*get_pbdma_signature)(struct gk20a *g);
657 int (*set_runlist_interleave)(struct gk20a *g, u32 id,
658 u32 runlist_id,
659 u32 new_level);
660 int (*tsg_set_timeslice)(struct tsg_gk20a *tsg, u32 timeslice);
661 u32 (*default_timeslice_us)(struct gk20a *);
662 int (*force_reset_ch)(struct channel_gk20a *ch,
663 u32 err_code, bool verbose);
664 int (*engine_enum_from_type)(struct gk20a *g, u32 engine_type,
665 u32 *inst_id);
666 void (*device_info_data_parse)(struct gk20a *g,
667 u32 table_entry, u32 *inst_id,
668 u32 *pri_base, u32 *fault_id);
669 u32 (*device_info_fault_id)(u32 table_entry);
670 int (*tsg_bind_channel)(struct tsg_gk20a *tsg,
671 struct channel_gk20a *ch);
672 int (*tsg_unbind_channel)(struct channel_gk20a *ch);
673 int (*tsg_open)(struct tsg_gk20a *tsg);
674 void (*tsg_release)(struct tsg_gk20a *tsg);
675 u32 (*eng_runlist_base_size)(void);
676 int (*init_engine_info)(struct fifo_gk20a *f);
677 u32 (*runlist_entry_size)(void);
678 void (*get_tsg_runlist_entry)(struct tsg_gk20a *tsg,
679 u32 *runlist);
680 void (*get_ch_runlist_entry)(struct channel_gk20a *ch,
681 u32 *runlist);
682 u32 (*userd_gp_get)(struct gk20a *g, struct channel_gk20a *ch);
683 void (*userd_gp_put)(struct gk20a *g, struct channel_gk20a *ch);
684 u64 (*userd_pb_get)(struct gk20a *g, struct channel_gk20a *ch);
685 void (*free_channel_ctx_header)(struct channel_gk20a *ch);
686 bool (*is_fault_engine_subid_gpc)(struct gk20a *g,
687 u32 engine_subid);
688 void (*dump_pbdma_status)(struct gk20a *g,
689 struct gk20a_debug_output *o);
690 void (*dump_eng_status)(struct gk20a *g,
691 struct gk20a_debug_output *o);
692 void (*dump_channel_status_ramfc)(struct gk20a *g,
693 struct gk20a_debug_output *o, u32 chid,
694 struct ch_state *ch_state);
695 u32 (*intr_0_error_mask)(struct gk20a *g);
696 int (*is_preempt_pending)(struct gk20a *g, u32 id,
697 unsigned int id_type);
698 void (*init_pbdma_intr_descs)(struct fifo_gk20a *f);
699 int (*reset_enable_hw)(struct gk20a *g);
700 int (*setup_userd)(struct channel_gk20a *c);
701 u32 (*pbdma_acquire_val)(u64 timeout);
702 void (*teardown_ch_tsg)(struct gk20a *g, u32 act_eng_bitmask,
703 u32 id, unsigned int id_type, unsigned int rc_type,
704 struct mmu_fault_info *mmfault);
705 bool (*handle_sched_error)(struct gk20a *g);
706 bool (*handle_ctxsw_timeout)(struct gk20a *g, u32 fifo_intr);
707 unsigned int (*handle_pbdma_intr_0)(struct gk20a *g,
708 u32 pbdma_id, u32 pbdma_intr_0,
709 u32 *handled, u32 *error_notifier);
710 unsigned int (*handle_pbdma_intr_1)(struct gk20a *g,
711 u32 pbdma_id, u32 pbdma_intr_1,
712 u32 *handled, u32 *error_notifier);
713 void (*init_eng_method_buffers)(struct gk20a *g,
714 struct tsg_gk20a *tsg);
715 void (*deinit_eng_method_buffers)(struct gk20a *g,
716 struct tsg_gk20a *tsg);
717 u32 (*get_preempt_timeout)(struct gk20a *g);
718 void (*post_event_id)(struct tsg_gk20a *tsg, int event_id);
719 void (*ch_abort_clean_up)(struct channel_gk20a *ch);
720 bool (*check_tsg_ctxsw_timeout)(struct tsg_gk20a *tsg,
721 bool *verbose, u32 *ms);
722 bool (*check_ch_ctxsw_timeout)(struct channel_gk20a *ch,
723 bool *verbose, u32 *ms);
724 int (*channel_suspend)(struct gk20a *g);
725 int (*channel_resume)(struct gk20a *g);
726 void (*set_error_notifier)(struct channel_gk20a *ch, u32 error);
727#ifdef CONFIG_TEGRA_GK20A_NVHOST
728 int (*alloc_syncpt_buf)(struct channel_gk20a *c,
729 u32 syncpt_id, struct nvgpu_mem *syncpt_buf);
730 void (*free_syncpt_buf)(struct channel_gk20a *c,
731 struct nvgpu_mem *syncpt_buf);
732 void (*add_syncpt_wait_cmd)(struct gk20a *g,
733 struct priv_cmd_entry *cmd, u32 off,
734 u32 id, u32 thresh, u64 gpu_va);
735 u32 (*get_syncpt_wait_cmd_size)(void);
736 void (*add_syncpt_incr_cmd)(struct gk20a *g,
737 bool wfi_cmd, struct priv_cmd_entry *cmd,
738 u32 id, u64 gpu_va);
739 u32 (*get_syncpt_incr_cmd_size)(bool wfi_cmd);
740 int (*get_sync_ro_map)(struct vm_gk20a *vm,
741 u64 *base_gpuva, u32 *sync_size);
742 u32 (*get_syncpt_incr_per_release)(void);
743#endif
744 void (*runlist_hw_submit)(struct gk20a *g, u32 runlist_id,
745 u32 count, u32 buffer_index);
746 int (*runlist_wait_pending)(struct gk20a *g, u32 runlist_id);
747 void (*ring_channel_doorbell)(struct channel_gk20a *c);
748 u32 (*get_sema_wait_cmd_size)(void);
749 u32 (*get_sema_incr_cmd_size)(void);
750 void (*add_sema_cmd)(struct gk20a *g,
751 struct nvgpu_semaphore *s, u64 sema_va,
752 struct priv_cmd_entry *cmd,
753 u32 off, bool acquire, bool wfi);
754 } fifo;
755 struct pmu_v {
756 u32 (*get_pmu_cmdline_args_size)(struct nvgpu_pmu *pmu);
757 void (*set_pmu_cmdline_args_cpu_freq)(struct nvgpu_pmu *pmu,
758 u32 freq);
759 void (*set_pmu_cmdline_args_trace_size)(struct nvgpu_pmu *pmu,
760 u32 size);
761 void (*set_pmu_cmdline_args_trace_dma_base)(
762 struct nvgpu_pmu *pmu);
763 void (*config_pmu_cmdline_args_super_surface)(
764 struct nvgpu_pmu *pmu);
765 void (*set_pmu_cmdline_args_trace_dma_idx)(
766 struct nvgpu_pmu *pmu, u32 idx);
767 void * (*get_pmu_cmdline_args_ptr)(struct nvgpu_pmu *pmu);
768 u32 (*get_pmu_allocation_struct_size)(struct nvgpu_pmu *pmu);
769 void (*set_pmu_allocation_ptr)(struct nvgpu_pmu *pmu,
770 void **pmu_alloc_ptr, void *assign_ptr);
771 void (*pmu_allocation_set_dmem_size)(struct nvgpu_pmu *pmu,
772 void *pmu_alloc_ptr, u16 size);
773 u16 (*pmu_allocation_get_dmem_size)(struct nvgpu_pmu *pmu,
774 void *pmu_alloc_ptr);
775 u32 (*pmu_allocation_get_dmem_offset)(struct nvgpu_pmu *pmu,
776 void *pmu_alloc_ptr);
777 u32 * (*pmu_allocation_get_dmem_offset_addr)(
778 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr);
779 void (*pmu_allocation_set_dmem_offset)(struct nvgpu_pmu *pmu,
780 void *pmu_alloc_ptr, u32 offset);
781 void * (*pmu_allocation_get_fb_addr)(
782 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr);
783 u32 (*pmu_allocation_get_fb_size)(
784 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr);
785 void (*get_pmu_init_msg_pmu_queue_params)(
786 struct nvgpu_falcon_queue *queue, u32 id,
787 void *pmu_init_msg);
788 void *(*get_pmu_msg_pmu_init_msg_ptr)(
789 struct pmu_init_msg *init);
790 u16 (*get_pmu_init_msg_pmu_sw_mg_off)(
791 union pmu_init_msg_pmu *init_msg);
792 u16 (*get_pmu_init_msg_pmu_sw_mg_size)(
793 union pmu_init_msg_pmu *init_msg);
794 u32 (*get_pmu_perfmon_cmd_start_size)(void);
795 int (*get_perfmon_cmd_start_offsetofvar)(
796 enum pmu_perfmon_cmd_start_fields field);
797 void (*perfmon_start_set_cmd_type)(struct pmu_perfmon_cmd *pc,
798 u8 value);
799 void (*perfmon_start_set_group_id)(struct pmu_perfmon_cmd *pc,
800 u8 value);
801 void (*perfmon_start_set_state_id)(struct pmu_perfmon_cmd *pc,
802 u8 value);
803 void (*perfmon_start_set_flags)(struct pmu_perfmon_cmd *pc,
804 u8 value);
805 u8 (*perfmon_start_get_flags)(struct pmu_perfmon_cmd *pc);
806 u32 (*get_pmu_perfmon_cmd_init_size)(void);
807 int (*get_perfmon_cmd_init_offsetofvar)(
808 enum pmu_perfmon_cmd_start_fields field);
809 void (*perfmon_cmd_init_set_sample_buffer)(
810 struct pmu_perfmon_cmd *pc, u16 value);
811 void (*perfmon_cmd_init_set_dec_cnt)(
812 struct pmu_perfmon_cmd *pc, u8 value);
813 void (*perfmon_cmd_init_set_base_cnt_id)(
814 struct pmu_perfmon_cmd *pc, u8 value);
815 void (*perfmon_cmd_init_set_samp_period_us)(
816 struct pmu_perfmon_cmd *pc, u32 value);
817 void (*perfmon_cmd_init_set_num_cnt)(struct pmu_perfmon_cmd *pc,
818 u8 value);
819 void (*perfmon_cmd_init_set_mov_avg)(struct pmu_perfmon_cmd *pc,
820 u8 value);
821 void *(*get_pmu_seq_in_a_ptr)(
822 struct pmu_sequence *seq);
823 void *(*get_pmu_seq_out_a_ptr)(
824 struct pmu_sequence *seq);
825 void (*set_pmu_cmdline_args_secure_mode)(struct nvgpu_pmu *pmu,
826 u32 val);
827 u32 (*get_perfmon_cntr_sz)(struct nvgpu_pmu *pmu);
828 void * (*get_perfmon_cntr_ptr)(struct nvgpu_pmu *pmu);
829 void (*set_perfmon_cntr_ut)(struct nvgpu_pmu *pmu, u16 ut);
830 void (*set_perfmon_cntr_lt)(struct nvgpu_pmu *pmu, u16 lt);
831 void (*set_perfmon_cntr_valid)(struct nvgpu_pmu *pmu, u8 val);
832 void (*set_perfmon_cntr_index)(struct nvgpu_pmu *pmu, u8 val);
833 void (*set_perfmon_cntr_group_id)(struct nvgpu_pmu *pmu,
834 u8 gid);
835
836 u8 (*pg_cmd_eng_buf_load_size)(struct pmu_pg_cmd *pg);
837 void (*pg_cmd_eng_buf_load_set_cmd_type)(struct pmu_pg_cmd *pg,
838 u8 value);
839 void (*pg_cmd_eng_buf_load_set_engine_id)(struct pmu_pg_cmd *pg,
840 u8 value);
841 void (*pg_cmd_eng_buf_load_set_buf_idx)(struct pmu_pg_cmd *pg,
842 u8 value);
843 void (*pg_cmd_eng_buf_load_set_pad)(struct pmu_pg_cmd *pg,
844 u8 value);
845 void (*pg_cmd_eng_buf_load_set_buf_size)(struct pmu_pg_cmd *pg,
846 u16 value);
847 void (*pg_cmd_eng_buf_load_set_dma_base)(struct pmu_pg_cmd *pg,
848 u32 value);
849 void (*pg_cmd_eng_buf_load_set_dma_offset)(struct pmu_pg_cmd *pg,
850 u8 value);
851 void (*pg_cmd_eng_buf_load_set_dma_idx)(struct pmu_pg_cmd *pg,
852 u8 value);
853 struct {
854 int (*boardobjgrp_pmucmd_construct_impl)
855 (struct gk20a *g,
856 struct boardobjgrp *pboardobjgrp,
857 struct boardobjgrp_pmu_cmd *cmd, u8 id, u8 msgid,
858 u8 hdrsize, u8 entrysize, u16 fbsize, u32 ss_offset,
859 u8 rpc_func_id);
860 int (*boardobjgrp_pmuset_impl)(struct gk20a *g,
861 struct boardobjgrp *pboardobjgrp);
862 int (*boardobjgrp_pmugetstatus_impl)(struct gk20a *g,
863 struct boardobjgrp *pboardobjgrp,
864 struct boardobjgrpmask *mask);
865 int (*is_boardobjgrp_pmucmd_id_valid)(struct gk20a *g,
866 struct boardobjgrp *pboardobjgrp,
867 struct boardobjgrp_pmu_cmd *cmd);
868 } boardobj;
869 struct {
870 u32 (*volt_set_voltage)(struct gk20a *g,
871 u32 logic_voltage_uv, u32 sram_voltage_uv);
872 u32 (*volt_get_voltage)(struct gk20a *g,
873 u8 volt_domain, u32 *pvoltage_uv);
874 u32 (*volt_send_load_cmd_to_pmu)(struct gk20a *g);
875 } volt;
876 struct {
877 u32 (*get_vbios_clk_domain)(u32 vbios_domain);
878 u32 (*clk_avfs_get_vin_cal_data)(struct gk20a *g,
879 struct avfsvinobjs *pvinobjs,
880 struct vin_device_v20 *pvindev);
881 u32 (*clk_vf_change_inject_data_fill)(struct gk20a *g,
882 struct nv_pmu_clk_rpc *rpccall,
883 struct set_fll_clk *setfllclk);
884 u32 (*perf_pmu_vfe_load)(struct gk20a *g);
885 u32 (*clk_set_boot_clk)(struct gk20a *g);
886 }clk;
887 } pmu_ver;
888 struct {
889 int (*get_netlist_name)(struct gk20a *g, int index, char *name);
890 bool (*is_fw_defined)(void);
891 } gr_ctx;
892#ifdef CONFIG_GK20A_CTXSW_TRACE
893 /*
894 * Currently only supported on Linux due to the extremely tight
895 * integration with Linux device driver structure (in particular
896 * mmap).
897 */
898 struct {
899 int (*init)(struct gk20a *g);
900 int (*max_entries)(struct gk20a *,
901 struct nvgpu_gpu_ctxsw_trace_filter *filter);
902 int (*flush)(struct gk20a *g);
903 int (*poll)(struct gk20a *g);
904 int (*enable)(struct gk20a *g);
905 int (*disable)(struct gk20a *g);
906 bool (*is_enabled)(struct gk20a *g);
907 int (*reset)(struct gk20a *g);
908 int (*bind_channel)(struct gk20a *g, struct channel_gk20a *ch);
909 int (*unbind_channel)(struct gk20a *g,
910 struct channel_gk20a *ch);
911 int (*deinit)(struct gk20a *g);
912 int (*alloc_user_buffer)(struct gk20a *g,
913 void **buf, size_t *size);
914 int (*free_user_buffer)(struct gk20a *g);
915 int (*mmap_user_buffer)(struct gk20a *g,
916 struct vm_area_struct *vma);
917 int (*set_filter)(struct gk20a *g,
918 struct nvgpu_gpu_ctxsw_trace_filter *filter);
919 } fecs_trace;
920#endif
921 struct {
922 bool (*support_sparse)(struct gk20a *g);
923 u64 (*gmmu_map)(struct vm_gk20a *vm,
924 u64 map_offset,
925 struct nvgpu_sgt *sgt,
926 u64 buffer_offset,
927 u64 size,
928 u32 pgsz_idx,
929 u8 kind_v,
930 u32 ctag_offset,
931 u32 flags,
932 enum gk20a_mem_rw_flag rw_flag,
933 bool clear_ctags,
934 bool sparse,
935 bool priv,
936 struct vm_gk20a_mapping_batch *batch,
937 enum nvgpu_aperture aperture);
938 void (*gmmu_unmap)(struct vm_gk20a *vm,
939 u64 vaddr,
940 u64 size,
941 u32 pgsz_idx,
942 bool va_allocated,
943 enum gk20a_mem_rw_flag rw_flag,
944 bool sparse,
945 struct vm_gk20a_mapping_batch *batch);
946 int (*vm_bind_channel)(struct vm_gk20a *vm,
947 struct channel_gk20a *ch);
948 int (*fb_flush)(struct gk20a *g);
949 void (*l2_invalidate)(struct gk20a *g);
950 void (*l2_flush)(struct gk20a *g, bool invalidate);
951 void (*cbc_clean)(struct gk20a *g);
952 void (*set_big_page_size)(struct gk20a *g,
953 struct nvgpu_mem *mem, int size);
954 u32 (*get_big_page_sizes)(void);
955 u32 (*get_default_big_page_size)(void);
956 u32 (*get_iommu_bit)(struct gk20a *g);
957 int (*init_mm_setup_hw)(struct gk20a *g);
958 bool (*is_bar1_supported)(struct gk20a *g);
959 int (*init_bar2_vm)(struct gk20a *g);
960 void (*remove_bar2_vm)(struct gk20a *g);
961 const struct gk20a_mmu_level *
962 (*get_mmu_levels)(struct gk20a *g, u32 big_page_size);
963 void (*init_pdb)(struct gk20a *g, struct nvgpu_mem *inst_block,
964 struct vm_gk20a *vm);
965 u64 (*gpu_phys_addr)(struct gk20a *g,
966 struct nvgpu_gmmu_attrs *attrs, u64 phys);
967 int (*alloc_inst_block)(struct gk20a *g,
968 struct nvgpu_mem *inst_block);
969 void (*init_inst_block)(struct nvgpu_mem *inst_block,
970 struct vm_gk20a *vm, u32 big_page_size);
971 bool (*mmu_fault_pending)(struct gk20a *g);
972 void (*fault_info_mem_destroy)(struct gk20a *g);
973 void (*mmu_fault_disable_hw)(struct gk20a *g);
974 u32 (*get_kind_invalid)(void);
975 u32 (*get_kind_pitch)(void);
976 u32 (*get_flush_retries)(struct gk20a *g,
977 enum nvgpu_flush_op op);
978 } mm;
979 /*
980 * This function is called to allocate secure memory (memory
981 * that the CPU cannot see). The function should fill the
982 * context buffer descriptor (especially fields destroy, sgt,
983 * size).
984 */
985 int (*secure_alloc)(struct gk20a *g,
986 struct gr_ctx_buffer_desc *desc,
987 size_t size);
988 struct {
989 void (*exit)(struct gk20a *g, struct nvgpu_mem *mem,
990 struct nvgpu_sgl *sgl);
991 u32 (*data032_r)(u32 i);
992 } pramin;
993 struct {
994 int (*init_therm_setup_hw)(struct gk20a *g);
995 void (*init_elcg_mode)(struct gk20a *g, u32 mode, u32 engine);
996 void (*init_blcg_mode)(struct gk20a *g, u32 mode, u32 engine);
997 int (*elcg_init_idle_filters)(struct gk20a *g);
998#ifdef CONFIG_DEBUG_FS
999 void (*therm_debugfs_init)(struct gk20a *g);
1000#endif
1001 int (*get_internal_sensor_curr_temp)(struct gk20a *g, u32 *temp_f24_8);
1002 void (*get_internal_sensor_limits)(s32 *max_24_8,
1003 s32 *min_24_8);
1004 u32 (*configure_therm_alert)(struct gk20a *g, s32 curr_warn_temp);
1005 } therm;
1006 struct {
1007 bool (*is_pmu_supported)(struct gk20a *g);
1008 int (*prepare_ucode)(struct gk20a *g);
1009 int (*pmu_setup_hw_and_bootstrap)(struct gk20a *g);
1010 int (*pmu_nsbootstrap)(struct nvgpu_pmu *pmu);
1011 int (*pmu_init_perfmon)(struct nvgpu_pmu *pmu);
1012 int (*pmu_perfmon_start_sampling)(struct nvgpu_pmu *pmu);
1013 int (*pmu_perfmon_stop_sampling)(struct nvgpu_pmu *pmu);
1014 int (*pmu_perfmon_get_samples_rpc)(struct nvgpu_pmu *pmu);
1015 int (*pmu_setup_elpg)(struct gk20a *g);
1016 u32 (*pmu_get_queue_head)(u32 i);
1017 u32 (*pmu_get_queue_head_size)(void);
1018 u32 (*pmu_get_queue_tail_size)(void);
1019 u32 (*pmu_get_queue_tail)(u32 i);
1020 int (*pmu_queue_head)(struct gk20a *g,
1021 struct nvgpu_falcon_queue *queue, u32 *head, bool set);
1022 int (*pmu_queue_tail)(struct gk20a *g,
1023 struct nvgpu_falcon_queue *queue, u32 *tail, bool set);
1024 void (*pmu_msgq_tail)(struct nvgpu_pmu *pmu,
1025 u32 *tail, bool set);
1026 u32 (*pmu_mutex_size)(void);
1027 int (*pmu_mutex_acquire)(struct nvgpu_pmu *pmu,
1028 u32 id, u32 *token);
1029 int (*pmu_mutex_release)(struct nvgpu_pmu *pmu,
1030 u32 id, u32 *token);
1031 bool (*pmu_is_interrupted)(struct nvgpu_pmu *pmu);
1032 void (*pmu_isr)(struct gk20a *g);
1033 void (*pmu_init_perfmon_counter)(struct gk20a *g);
1034 void (*pmu_pg_idle_counter_config)(struct gk20a *g, u32 pg_engine_id);
1035 u32 (*pmu_read_idle_counter)(struct gk20a *g, u32 counter_id);
1036 void (*pmu_reset_idle_counter)(struct gk20a *g, u32 counter_id);
1037 void (*pmu_dump_elpg_stats)(struct nvgpu_pmu *pmu);
1038 void (*pmu_dump_falcon_stats)(struct nvgpu_pmu *pmu);
1039 void (*pmu_enable_irq)(struct nvgpu_pmu *pmu, bool enable);
1040 int (*init_wpr_region)(struct gk20a *g);
1041 int (*load_lsfalcon_ucode)(struct gk20a *g, u32 falconidmask);
1042 void (*write_dmatrfbase)(struct gk20a *g, u32 addr);
1043 void (*pmu_elpg_statistics)(struct gk20a *g, u32 pg_engine_id,
1044 struct pmu_pg_stats_data *pg_stat_data);
1045 int (*pmu_pg_init_param)(struct gk20a *g, u32 pg_engine_id);
1046 int (*pmu_pg_set_sub_feature_mask)(struct gk20a *g,
1047 u32 pg_engine_id);
1048 u32 (*pmu_pg_supported_engines_list)(struct gk20a *g);
1049 u32 (*pmu_pg_engines_feature_list)(struct gk20a *g,
1050 u32 pg_engine_id);
1051 bool (*pmu_is_lpwr_feature_supported)(struct gk20a *g,
1052 u32 feature_id);
1053 int (*pmu_lpwr_enable_pg)(struct gk20a *g, bool pstate_lock);
1054 int (*pmu_lpwr_disable_pg)(struct gk20a *g, bool pstate_lock);
1055 u32 (*pmu_pg_param_post_init)(struct gk20a *g);
1056 void (*dump_secure_fuses)(struct gk20a *g);
1057 int (*reset_engine)(struct gk20a *g, bool do_reset);
1058 bool (*is_engine_in_reset)(struct gk20a *g);
1059 int (*falcon_wait_for_halt)(struct gk20a *g,
1060 unsigned int timeout);
1061 int (*falcon_clear_halt_interrupt_status)(struct gk20a *g,
1062 unsigned int timeout);
1063 int (*init_falcon_setup_hw)(struct gk20a *g,
1064 void *desc, u32 bl_sz);
1065 bool (*is_lazy_bootstrap)(u32 falcon_id);
1066 bool (*is_priv_load)(u32 falcon_id);
1067 void (*get_wpr)(struct gk20a *g, struct wpr_carveout_info *inf);
1068 int (*alloc_blob_space)(struct gk20a *g,
1069 size_t size, struct nvgpu_mem *mem);
1070 int (*pmu_populate_loader_cfg)(struct gk20a *g,
1071 void *lsfm, u32 *p_bl_gen_desc_size);
1072 int (*flcn_populate_bl_dmem_desc)(struct gk20a *g,
1073 void *lsfm, u32 *p_bl_gen_desc_size, u32 falconid);
1074 void (*handle_ext_irq)(struct gk20a *g, u32 intr);
1075 void (*set_irqmask)(struct gk20a *g);
1076 void (*update_lspmu_cmdline_args)(struct gk20a *g);
1077 void (*setup_apertures)(struct gk20a *g);
1078 u32 (*get_irqdest)(struct gk20a *g);
1079 int (*alloc_super_surface)(struct gk20a *g,
1080 struct nvgpu_mem *super_surface, u32 size);
1081 bool (*is_debug_mode_enabled)(struct gk20a *g);
1082 } pmu;
1083 struct {
1084 int (*init_debugfs)(struct gk20a *g);
1085 void (*disable_slowboot)(struct gk20a *g);
1086 int (*init_clk_support)(struct gk20a *g);
1087 int (*suspend_clk_support)(struct gk20a *g);
1088 u32 (*get_crystal_clk_hz)(struct gk20a *g);
1089 int (*clk_domain_get_f_points)(struct gk20a *g,
1090 u32 clkapidomain, u32 *pfpointscount,
1091 u16 *pfreqpointsinmhz);
1092 unsigned long (*measure_freq)(struct gk20a *g, u32 api_domain);
1093 u32 (*get_rate_cntr)(struct gk20a *g, struct namemap_cfg *c);
1094 unsigned long (*get_rate)(struct gk20a *g, u32 api_domain);
1095 int (*set_rate)(struct gk20a *g, u32 api_domain, unsigned long rate);
1096 unsigned long (*get_fmax_at_vmin_safe)(struct gk20a *g);
1097 u32 (*get_ref_clock_rate)(struct gk20a *g);
1098 int (*predict_mv_at_hz_cur_tfloor)(struct clk_gk20a *clk,
1099 unsigned long rate);
1100 unsigned long (*get_maxrate)(struct gk20a *g, u32 api_domain);
1101 int (*prepare_enable)(struct clk_gk20a *clk);
1102 void (*disable_unprepare)(struct clk_gk20a *clk);
1103 int (*get_voltage)(struct clk_gk20a *clk, u64 *val);
1104 int (*get_gpcclk_clock_counter)(struct clk_gk20a *clk, u64 *val);
1105 int (*pll_reg_write)(struct gk20a *g, u32 reg, u32 val);
1106 int (*get_pll_debug_data)(struct gk20a *g,
1107 struct nvgpu_clk_pll_debug_data *d);
1108 int (*mclk_init)(struct gk20a *g);
1109 void (*mclk_deinit)(struct gk20a *g);
1110 int (*mclk_change)(struct gk20a *g, u16 val);
1111 bool split_rail_support;
1112 bool support_clk_freq_controller;
1113 bool support_pmgr_domain;
1114 bool support_lpwr_pg;
1115 } clk;
1116 struct {
1117 int (*arbiter_clk_init)(struct gk20a *g);
1118 u32 (*get_arbiter_clk_domains)(struct gk20a *g);
1119 int (*get_arbiter_clk_range)(struct gk20a *g, u32 api_domain,
1120 u16 *min_mhz, u16 *max_mhz);
1121 int (*get_arbiter_clk_default)(struct gk20a *g, u32 api_domain,
1122 u16 *default_mhz);
1123 void (*clk_arb_run_arbiter_cb)(struct nvgpu_clk_arb *arb);
1124 /* This function is inherently unsafe to call while
1125 * arbiter is running arbiter must be blocked
1126 * before calling this function */
1127 int (*get_current_pstate)(struct gk20a *g);
1128 void (*clk_arb_cleanup)(struct nvgpu_clk_arb *arb);
1129 } clk_arb;
1130 struct {
1131 int (*handle_pmu_perf_event)(struct gk20a *g, void *pmu_msg);
1132 } perf;
1133 struct {
1134 int (*exec_regops)(struct dbg_session_gk20a *dbg_s,
1135 struct nvgpu_dbg_reg_op *ops,
1136 u64 num_ops,
1137 bool *is_current_ctx);
1138 const struct regop_offset_range* (
1139 *get_global_whitelist_ranges)(void);
1140 u64 (*get_global_whitelist_ranges_count)(void);
1141 const struct regop_offset_range* (
1142 *get_context_whitelist_ranges)(void);
1143 u64 (*get_context_whitelist_ranges_count)(void);
1144 const u32* (*get_runcontrol_whitelist)(void);
1145 u64 (*get_runcontrol_whitelist_count)(void);
1146 const u32* (*get_qctl_whitelist)(void);
1147 u64 (*get_qctl_whitelist_count)(void);
1148 } regops;
1149 struct {
1150 void (*intr_mask)(struct gk20a *g);
1151 void (*intr_enable)(struct gk20a *g);
1152 void (*intr_unit_config)(struct gk20a *g,
1153 bool enable, bool is_stalling, u32 unit);
1154 void (*isr_stall)(struct gk20a *g);
1155 bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr);
1156 bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr);
1157 bool (*is_stall_and_eng_intr_pending)(struct gk20a *g,
1158 u32 act_eng_id, u32 *eng_intr_pending);
1159 u32 (*intr_stall)(struct gk20a *g);
1160 void (*intr_stall_pause)(struct gk20a *g);
1161 void (*intr_stall_resume)(struct gk20a *g);
1162 u32 (*intr_nonstall)(struct gk20a *g);
1163 void (*intr_nonstall_pause)(struct gk20a *g);
1164 void (*intr_nonstall_resume)(struct gk20a *g);
1165 u32 (*isr_nonstall)(struct gk20a *g);
1166 void (*enable)(struct gk20a *g, u32 units);
1167 void (*disable)(struct gk20a *g, u32 units);
1168 void (*reset)(struct gk20a *g, u32 units);
1169 u32 (*boot_0)(struct gk20a *g, u32 *arch, u32 *impl, u32 *rev);
1170 bool (*is_intr1_pending)(struct gk20a *g, enum nvgpu_unit unit, u32 mc_intr_1);
1171 void (*log_pending_intrs)(struct gk20a *g);
1172 } mc;
1173 struct {
1174 void (*show_dump)(struct gk20a *g,
1175 struct gk20a_debug_output *o);
1176 } debug;
1177 struct {
1178 int (*dbg_set_powergate)(struct dbg_session_gk20a *dbg_s,
1179 bool disable_powergate);
1180 bool (*check_and_set_global_reservation)(
1181 struct dbg_session_gk20a *dbg_s,
1182 struct dbg_profiler_object_data *prof_obj);
1183 bool (*check_and_set_context_reservation)(
1184 struct dbg_session_gk20a *dbg_s,
1185 struct dbg_profiler_object_data *prof_obj);
1186 void (*release_profiler_reservation)(
1187 struct dbg_session_gk20a *dbg_s,
1188 struct dbg_profiler_object_data *prof_obj);
1189 int (*perfbuffer_enable)(struct gk20a *g, u64 offset, u32 size);
1190 int (*perfbuffer_disable)(struct gk20a *g);
1191 } dbg_session_ops;
1192
1193 int (*get_litter_value)(struct gk20a *g, int value);
1194 int (*chip_init_gpu_characteristics)(struct gk20a *g);
1195
1196 struct {
1197 void (*init_hw)(struct gk20a *g);
1198 void (*isr)(struct gk20a *g);
1199 int (*bar1_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst);
1200 int (*bar2_bind)(struct gk20a *g, struct nvgpu_mem *bar1_inst);
1201 u32 (*set_bar0_window)(struct gk20a *g, struct nvgpu_mem *mem,
1202 struct nvgpu_sgt *sgt, struct nvgpu_sgl *sgl,
1203 u32 w);
1204 u32 (*read_sw_scratch)(struct gk20a *g, u32 index);
1205 void (*write_sw_scratch)(struct gk20a *g, u32 index, u32 val);
1206 } bus;
1207
1208 struct {
1209 void (*isr)(struct gk20a *g);
1210 int (*read_ptimer)(struct gk20a *g, u64 *value);
1211 int (*get_timestamps_zipper)(struct gk20a *g,
1212 u32 source_id, u32 count,
1213 struct nvgpu_cpu_time_correlation_sample *);
1214 } ptimer;
1215
1216 struct {
1217 int (*init)(struct gk20a *g);
1218 int (*preos_wait_for_halt)(struct gk20a *g);
1219 void (*preos_reload_check)(struct gk20a *g);
1220 int (*devinit)(struct gk20a *g);
1221 int (*preos)(struct gk20a *g);
1222 int (*verify_devinit)(struct gk20a *g);
1223 } bios;
1224
1225#if defined(CONFIG_GK20A_CYCLE_STATS)
1226 struct {
1227 int (*enable_snapshot)(struct channel_gk20a *ch,
1228 struct gk20a_cs_snapshot_client *client);
1229 void (*disable_snapshot)(struct gr_gk20a *gr);
1230 int (*check_data_available)(struct channel_gk20a *ch,
1231 u32 *pending,
1232 bool *hw_overflow);
1233 void (*set_handled_snapshots)(struct gk20a *g, u32 num);
1234 u32 (*allocate_perfmon_ids)(struct gk20a_cs_snapshot *data,
1235 u32 count);
1236 u32 (*release_perfmon_ids)(struct gk20a_cs_snapshot *data,
1237 u32 start,
1238 u32 count);
1239 int (*detach_snapshot)(struct channel_gk20a *ch,
1240 struct gk20a_cs_snapshot_client *client);
1241 bool (*get_overflow_status)(struct gk20a *g);
1242 u32 (*get_pending_snapshots)(struct gk20a *g);
1243 } css;
1244#endif
1245 struct {
1246 int (*get_speed)(struct gk20a *g, u32 *xve_link_speed);
1247 int (*set_speed)(struct gk20a *g, u32 xve_link_speed);
1248 void (*available_speeds)(struct gk20a *g, u32 *speed_mask);
1249 u32 (*xve_readl)(struct gk20a *g, u32 reg);
1250 void (*xve_writel)(struct gk20a *g, u32 reg, u32 val);
1251 void (*disable_aspm)(struct gk20a *g);
1252 void (*reset_gpu)(struct gk20a *g);
1253#if defined(CONFIG_PCI_MSI)
1254 void (*rearm_msi)(struct gk20a *g);
1255#endif
1256 void (*enable_shadow_rom)(struct gk20a *g);
1257 void (*disable_shadow_rom)(struct gk20a *g);
1258 u32 (*get_link_control_status)(struct gk20a *g);
1259 } xve;
1260 struct {
1261 int (*falcon_hal_sw_init)(struct nvgpu_falcon *flcn);
1262 } falcon;
1263 struct {
1264 void (*enable_priv_ring)(struct gk20a *g);
1265 void (*isr)(struct gk20a *g);
1266 void (*decode_error_code)(struct gk20a *g, u32 error_code);
1267 void (*set_ppriv_timeout_settings)(struct gk20a *g);
1268 u32 (*enum_ltc)(struct gk20a *g);
1269 } priv_ring;
1270 struct {
1271 int (*check_priv_security)(struct gk20a *g);
1272 bool (*is_opt_ecc_enable)(struct gk20a *g);
1273 bool (*is_opt_feature_override_disable)(struct gk20a *g);
1274 u32 (*fuse_status_opt_fbio)(struct gk20a *g);
1275 u32 (*fuse_status_opt_fbp)(struct gk20a *g);
1276 u32 (*fuse_status_opt_rop_l2_fbp)(struct gk20a *g, u32 fbp);
1277 u32 (*fuse_status_opt_tpc_gpc)(struct gk20a *g, u32 gpc);
1278 void (*fuse_ctrl_opt_tpc_gpc)(struct gk20a *g, u32 gpc, u32 val);
1279 u32 (*fuse_opt_sec_debug_en)(struct gk20a *g);
1280 u32 (*fuse_opt_priv_sec_en)(struct gk20a *g);
1281 u32 (*read_vin_cal_fuse_rev)(struct gk20a *g);
1282 u32 (*read_vin_cal_slope_intercept_fuse)(struct gk20a *g,
1283 u32 vin_id, u32 *slope,
1284 u32 *intercept);
1285 u32 (*read_vin_cal_gain_offset_fuse)(struct gk20a *g,
1286 u32 vin_id, s8 *gain,
1287 s8 *offset);
1288 } fuse;
1289 struct {
1290 int (*init)(struct gk20a *g);
1291 int (*discover_ioctrl)(struct gk20a *g);
1292 int (*discover_link)(struct gk20a *g);
1293 int (*isr)(struct gk20a *g);
1294 int (*rxdet)(struct gk20a *g, u32 link_id);
1295 int (*setup_pll)(struct gk20a *g, unsigned long link_mask);
1296 int (*minion_data_ready_en)(struct gk20a *g,
1297 unsigned long link_mask, bool sync);
1298 void (*get_connected_link_mask)(u32 *link_mask);
1299 void (*set_sw_war)(struct gk20a *g, u32 link_id);
1300 /* API */
1301 int (*link_early_init)(struct gk20a *g, unsigned long mask);
1302 u32 (*link_get_mode)(struct gk20a *g, u32 link_id);
1303 u32 (*link_get_state)(struct gk20a *g, u32 link_id);
1304 int (*link_set_mode)(struct gk20a *g, u32 link_id, u32 mode);
1305 u32 (*get_sublink_mode)(struct gk20a *g, u32 link_id,
1306 bool is_rx_sublink);
1307 u32 (*get_rx_sublink_state)(struct gk20a *g, u32 link_id);
1308 u32 (*get_tx_sublink_state)(struct gk20a *g, u32 link_id);
1309 int (*set_sublink_mode)(struct gk20a *g, u32 link_id,
1310 bool is_rx_sublink, u32 mode);
1311 int (*interface_init)(struct gk20a *g);
1312 int (*interface_disable)(struct gk20a *g);
1313 int (*reg_init)(struct gk20a *g);
1314 int (*shutdown)(struct gk20a *g);
1315 int (*early_init)(struct gk20a *g);
1316 } nvlink;
1317 struct {
1318 u32 (*get_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g);
1319 void (*set_nvhsclk_ctrl_e_clk_nvl)(struct gk20a *g, u32 val);
1320 u32 (*get_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g);
1321 void (*set_nvhsclk_ctrl_swap_clk_nvl)(struct gk20a *g, u32 val);
1322 } top;
1323 void (*semaphore_wakeup)(struct gk20a *g, bool post_events);
1324};
1325
1326struct nvgpu_bios_ucode {
1327 u8 *bootloader;
1328 u32 bootloader_phys_base;
1329 u32 bootloader_size;
1330 u8 *ucode;
1331 u32 phys_base;
1332 u32 size;
1333 u8 *dmem;
1334 u32 dmem_phys_base;
1335 u32 dmem_size;
1336 u32 code_entry_point;
1337};
1338
1339struct nvgpu_bios {
1340 u32 vbios_version;
1341 u8 vbios_oem_version;
1342
1343 u8 *data;
1344 size_t size;
1345
1346 struct nvgpu_bios_ucode devinit;
1347 struct nvgpu_bios_ucode preos;
1348
1349 u8 *devinit_tables;
1350 u32 devinit_tables_size;
1351 u8 *bootscripts;
1352 u32 bootscripts_size;
1353
1354 u8 mem_strap_data_count;
1355 u16 mem_strap_xlat_tbl_ptr;
1356
1357 u32 condition_table_ptr;
1358
1359 u32 devinit_tables_phys_base;
1360 u32 devinit_script_phys_base;
1361
1362 struct bit_token *perf_token;
1363 struct bit_token *clock_token;
1364 struct bit_token *virt_token;
1365 u32 expansion_rom_offset;
1366
1367 u32 nvlink_config_data_offset;
1368};
1369
1370struct nvgpu_gpu_params {
1371 /* GPU architecture ID */
1372 u32 gpu_arch;
1373 /* GPU implementation ID */
1374 u32 gpu_impl;
1375 /* GPU revision ID */
1376 u32 gpu_rev;
1377 /* sm version */
1378 u32 sm_arch_sm_version;
1379 /* sm instruction set */
1380 u32 sm_arch_spa_version;
1381 u32 sm_arch_warp_count;
1382};
1383
1384struct gk20a {
1385 void (*free)(struct gk20a *g);
1386 struct nvgpu_nvhost_dev *nvhost_dev;
1387
1388 /*
1389 * Used by <nvgpu/enabled.h>. Do not access directly!
1390 */
1391 unsigned long *enabled_flags;
1392
1393 nvgpu_atomic_t usage_count;
1394
1395 struct nvgpu_mutex ctxsw_disable_lock;
1396 int ctxsw_disable_count;
1397
1398 struct nvgpu_ref refcount;
1399
1400 const char *name;
1401
1402 bool gpu_reset_done;
1403 bool power_on;
1404 bool suspended;
1405 bool sw_ready;
1406
1407 u64 log_mask;
1408 u32 log_trace;
1409
1410 struct nvgpu_mutex tpc_pg_lock;
1411
1412 struct nvgpu_gpu_params params;
1413
1414 /*
1415 * Guards access to hardware when usual gk20a_{busy,idle} are skipped
1416 * for submits and held for channel lifetime but dropped for an ongoing
1417 * gk20a_do_idle().
1418 */
1419 struct nvgpu_rwsem deterministic_busy;
1420
1421 struct nvgpu_falcon pmu_flcn;
1422 struct nvgpu_falcon sec2_flcn;
1423 struct nvgpu_falcon fecs_flcn;
1424 struct nvgpu_falcon gpccs_flcn;
1425 struct nvgpu_falcon nvdec_flcn;
1426 struct nvgpu_falcon minion_flcn;
1427 struct clk_gk20a clk;
1428 struct fifo_gk20a fifo;
1429 struct nvgpu_nvlink_dev nvlink;
1430 struct gr_gk20a gr;
1431 struct sim_nvgpu *sim;
1432 struct mm_gk20a mm;
1433 struct nvgpu_pmu pmu;
1434 struct acr_desc acr;
1435 struct nvgpu_ecc ecc;
1436 struct clk_pmupstate clk_pmu;
1437 struct perf_pmupstate perf_pmu;
1438 struct pmgr_pmupstate pmgr_pmu;
1439 struct therm_pmupstate therm_pmu;
1440
1441#ifdef CONFIG_DEBUG_FS
1442 struct railgate_stats pstats;
1443#endif
1444 u32 gr_idle_timeout_default;
1445 bool timeouts_disabled_by_user;
1446 unsigned int ch_wdt_timeout_ms;
1447 u32 fifo_eng_timeout_us;
1448
1449 struct nvgpu_mutex poweron_lock;
1450 struct nvgpu_mutex poweroff_lock;
1451
1452 /* Channel priorities */
1453 u32 timeslice_low_priority_us;
1454 u32 timeslice_medium_priority_us;
1455 u32 timeslice_high_priority_us;
1456 u32 min_timeslice_us;
1457 u32 max_timeslice_us;
1458 bool runlist_interleave;
1459
1460 bool slcg_enabled;
1461 bool blcg_enabled;
1462 bool elcg_enabled;
1463 bool elpg_enabled;
1464 bool aelpg_enabled;
1465 bool can_elpg;
1466 bool mscg_enabled;
1467 bool forced_idle;
1468 bool forced_reset;
1469 bool allow_all;
1470
1471 u32 ptimer_src_freq;
1472
1473 int railgate_delay;
1474 u8 ldiv_slowdown_factor;
1475 unsigned int aggressive_sync_destroy_thresh;
1476 bool aggressive_sync_destroy;
1477
1478 bool has_syncpoints;
1479 /* Debugfs knob for forcing syncpt support off in runtime. */
1480 u32 disable_syncpoints;
1481
1482 bool support_pmu;
1483 u32 bootstrap_owner;
1484
1485 bool is_virtual;
1486
1487 bool has_cde;
1488
1489 u32 emc3d_ratio;
1490
1491 struct nvgpu_spinlock ltc_enabled_lock;
1492
1493 struct gk20a_ctxsw_ucode_info ctxsw_ucode_info;
1494
1495 /*
1496 * A group of semaphore pools. One for each channel.
1497 */
1498 struct nvgpu_semaphore_sea *sema_sea;
1499
1500 /* held while manipulating # of debug/profiler sessions present */
1501 /* also prevents debug sessions from attaching until released */
1502 struct nvgpu_mutex dbg_sessions_lock;
1503 int dbg_powergating_disabled_refcount; /*refcount for pg disable */
1504 /*refcount for timeout disable */
1505 nvgpu_atomic_t timeouts_disabled_refcount;
1506
1507 /* must have dbg_sessions_lock before use */
1508 struct nvgpu_dbg_reg_op *dbg_regops_tmp_buf;
1509 u32 dbg_regops_tmp_buf_ops;
1510
1511 /* For perfbuf mapping */
1512 struct {
1513 struct dbg_session_gk20a *owner;
1514 u64 offset;
1515 } perfbuf;
1516
1517 /* For profiler reservations */
1518 struct nvgpu_list_node profiler_objects;
1519 bool global_profiler_reservation_held;
1520 int profiler_reservation_count;
1521
1522 void (*remove_support)(struct gk20a *);
1523
1524 u64 pg_ingating_time_us;
1525 u64 pg_ungating_time_us;
1526 u32 pg_gating_cnt;
1527
1528 struct nvgpu_spinlock mc_enable_lock;
1529
1530 struct gk20a_as as;
1531
1532 struct nvgpu_mutex client_lock;
1533 int client_refcount; /* open channels and ctrl nodes */
1534
1535 struct gpu_ops ops;
1536 u32 mc_intr_mask_restore[4];
1537 /*used for change of enum zbc update cmd id from ver 0 to ver1*/
1538 u32 pmu_ver_cmd_id_zbc_table_update;
1539 u32 pmu_lsf_pmu_wpr_init_done;
1540 u32 pmu_lsf_loaded_falcon_id;
1541
1542 int irqs_enabled;
1543 int irq_stall; /* can be same as irq_nonstall in case of PCI */
1544 int irq_nonstall;
1545 u32 max_ltc_count;
1546 u32 ltc_count;
1547 u32 ltc_streamid;
1548
1549 struct gk20a_worker {
1550 struct nvgpu_thread poll_task;
1551 nvgpu_atomic_t put;
1552 struct nvgpu_cond wq;
1553 struct nvgpu_list_node items;
1554 struct nvgpu_spinlock items_lock;
1555 struct nvgpu_mutex start_lock;
1556 } channel_worker, clk_arb_worker;
1557
1558 struct {
1559 void (*open)(struct channel_gk20a *ch);
1560 void (*close)(struct channel_gk20a *ch);
1561 void (*work_completion_signal)(struct channel_gk20a *ch);
1562 void (*work_completion_cancel_sync)(struct channel_gk20a *ch);
1563 bool (*os_fence_framework_inst_exists)(struct channel_gk20a *ch);
1564 int (*init_os_fence_framework)(
1565 struct channel_gk20a *ch, const char *fmt, ...);
1566 void (*signal_os_fence_framework)(struct channel_gk20a *ch);
1567 void (*destroy_os_fence_framework)(struct channel_gk20a *ch);
1568 int (*copy_user_gpfifo)(struct nvgpu_gpfifo_entry *dest,
1569 struct nvgpu_gpfifo_userdata userdata,
1570 u32 start, u32 length);
1571 int (*alloc_usermode_buffers)(struct channel_gk20a *c,
1572 struct nvgpu_gpfifo_args *gpfifo_args);
1573 } os_channel;
1574
1575 struct gk20a_scale_profile *scale_profile;
1576 unsigned long last_freq;
1577
1578 struct gk20a_ctxsw_trace *ctxsw_trace;
1579 struct gk20a_fecs_trace *fecs_trace;
1580
1581 bool mmu_debug_ctrl;
1582
1583 u32 tpc_fs_mask_user;
1584
1585 u32 tpc_pg_mask;
1586 bool can_tpc_powergate;
1587
1588 u32 valid_tpc_mask;
1589
1590 struct nvgpu_bios bios;
1591 bool bios_is_init;
1592
1593 struct nvgpu_clk_arb *clk_arb;
1594
1595 struct nvgpu_mutex clk_arb_enable_lock;
1596
1597 struct gk20a_ce_app ce_app;
1598
1599 bool ltc_intr_en_illegal_compstat;
1600
1601 /* PCI device identifier */
1602 u16 pci_vendor_id, pci_device_id;
1603 u16 pci_subsystem_vendor_id, pci_subsystem_device_id;
1604 u16 pci_class;
1605 u8 pci_revision;
1606
1607 /*
1608 * PCI power management: i2c device index, port and address for
1609 * INA3221.
1610 */
1611 u32 ina3221_dcb_index;
1612 u32 ina3221_i2c_address;
1613 u32 ina3221_i2c_port;
1614 bool hardcode_sw_threshold;
1615
1616 /* PCIe power states. */
1617 bool xve_l0s;
1618 bool xve_l1;
1619
1620 /* Current warning temp in sfxp24.8 */
1621 s32 curr_warn_temp;
1622
1623#if defined(CONFIG_PCI_MSI)
1624 /* Check if msi is enabled */
1625 bool msi_enabled;
1626#endif
1627#ifdef CONFIG_NVGPU_TRACK_MEM_USAGE
1628 struct nvgpu_mem_alloc_tracker *vmallocs;
1629 struct nvgpu_mem_alloc_tracker *kmallocs;
1630#endif
1631
1632 /* The minimum VBIOS version supported */
1633 u32 vbios_min_version;
1634
1635 /* memory training sequence and mclk switch scripts */
1636 u32 mem_config_idx;
1637
1638 u64 dma_memory_used;
1639
1640#if defined(CONFIG_TEGRA_GK20A_NVHOST)
1641 u64 syncpt_unit_base;
1642 size_t syncpt_unit_size;
1643 u32 syncpt_size;
1644#endif
1645 struct nvgpu_mem syncpt_mem;
1646
1647 struct nvgpu_list_node boardobj_head;
1648 struct nvgpu_list_node boardobjgrp_head;
1649};
1650
1651static inline bool nvgpu_is_timeouts_enabled(struct gk20a *g)
1652{
1653 return nvgpu_atomic_read(&g->timeouts_disabled_refcount) == 0;
1654}
1655
1656static inline u32 gk20a_get_gr_idle_timeout(struct gk20a *g)
1657{
1658 return nvgpu_is_timeouts_enabled(g) ?
1659 g->gr_idle_timeout_default : UINT_MAX;
1660}
1661
1662#define MULTICHAR_TAG(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d))
1663enum BAR0_DEBUG_OPERATION {
1664 BARO_ZERO_NOP = 0,
1665 OP_END = MULTICHAR_TAG('D', 'O', 'N', 'E'),
1666 BAR0_READ32 = MULTICHAR_TAG('0', 'R', '3', '2'),
1667 BAR0_WRITE32 = MULTICHAR_TAG('0', 'W', '3', '2'),
1668};
1669
1670struct share_buffer_head {
1671 enum BAR0_DEBUG_OPERATION operation;
1672/* size of the operation item */
1673 u32 size;
1674 u32 completed;
1675 u32 failed;
1676 u64 context;
1677 u64 completion_callback;
1678};
1679
1680struct gk20a_cyclestate_buffer_elem {
1681 struct share_buffer_head head;
1682/* in */
1683 u64 p_data;
1684 u64 p_done;
1685 u32 offset_bar0;
1686 u16 first_bit;
1687 u16 last_bit;
1688/* out */
1689/* keep 64 bits to be consistent */
1690 u64 data;
1691};
1692
1693/* operations that will need to be executed on non stall workqueue */
1694#define GK20A_NONSTALL_OPS_WAKEUP_SEMAPHORE BIT32(0)
1695#define GK20A_NONSTALL_OPS_POST_EVENTS BIT32(1)
1696
1697/* register accessors */
1698void __nvgpu_check_gpu_state(struct gk20a *g);
1699void __gk20a_warn_on_no_regs(void);
1700
1701/* classes that the device supports */
1702/* TBD: get these from an open-sourced SDK? */
1703enum {
1704 FERMI_TWOD_A = 0x902D,
1705 KEPLER_INLINE_TO_MEMORY_A = 0xA040,
1706 KEPLER_DMA_COPY_A = 0xA0B5,
1707};
1708
1709#define GK20A_BAR0_IORESOURCE_MEM 0
1710#define GK20A_BAR1_IORESOURCE_MEM 1
1711#define GK20A_SIM_IORESOURCE_MEM 2
1712
1713void gk20a_busy_noresume(struct gk20a *g);
1714void gk20a_idle_nosuspend(struct gk20a *g);
1715int __must_check gk20a_busy(struct gk20a *g);
1716void gk20a_idle(struct gk20a *g);
1717int __gk20a_do_idle(struct gk20a *g, bool force_reset);
1718int __gk20a_do_unidle(struct gk20a *g);
1719
1720int gk20a_can_busy(struct gk20a *g);
1721int gk20a_wait_for_idle(struct gk20a *g);
1722
1723#define NVGPU_GPU_ARCHITECTURE_SHIFT 4
1724
1725/* constructs unique and compact GPUID from nvgpu_gpu_characteristics
1726 * arch/impl fields */
1727#define GK20A_GPUID(arch, impl) ((u32) ((arch) | (impl)))
1728
1729#define GK20A_GPUID_GK20A 0x000000EA
1730#define GK20A_GPUID_GM20B 0x0000012B
1731#define GK20A_GPUID_GM20B_B 0x0000012E
1732#define NVGPU_GPUID_GP10B 0x0000013B
1733#define NVGPU_GPUID_GP104 0x00000134
1734#define NVGPU_GPUID_GP106 0x00000136
1735#define NVGPU_GPUID_GV11B 0x0000015B
1736#define NVGPU_GPUID_GV100 0x00000140
1737
1738int gk20a_init_gpu_characteristics(struct gk20a *g);
1739
1740int gk20a_prepare_poweroff(struct gk20a *g);
1741int gk20a_finalize_poweron(struct gk20a *g);
1742
1743void nvgpu_wait_for_deferred_interrupts(struct gk20a *g);
1744
1745struct gk20a * __must_check gk20a_get(struct gk20a *g);
1746void gk20a_put(struct gk20a *g);
1747
1748static inline bool gk20a_platform_has_syncpoints(struct gk20a *g)
1749{
1750#ifdef CONFIG_TEGRA_GK20A_NVHOST
1751 return g->has_syncpoints && !g->disable_syncpoints;
1752#else
1753 return false;
1754#endif
1755}
1756
1757int gk20a_detect_chip(struct gk20a *g);
1758#endif /* GK20A_H */
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-04 20:47:45 -0400