summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
blob: 92dcc8e66c00a8188fb4dd256e875f9e5db5974b (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
/*
 * drivers/video/tegra/host/gk20a/fifo_gk20a.h
 *
 * GK20A graphics fifo (gr host)
 *
 * Copyright (c) 2011-2017, NVIDIA CORPORATION.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 */
#ifndef __FIFO_GK20A_H__
#define __FIFO_GK20A_H__

#include "channel_gk20a.h"
#include "tsg_gk20a.h"

#ifdef CONFIG_TEGRA_19x_GPU
#include "fifo_t19x.h"
#endif

#include <nvgpu/kref.h>

struct gk20a_debug_output;

#define MAX_RUNLIST_BUFFERS		2

#define FIFO_INVAL_ENGINE_ID		((u32)~0)
#define FIFO_INVAL_CHANNEL_ID		((u32)~0)
#define FIFO_INVAL_TSG_ID		((u32)~0)

#define ID_TYPE_CHANNEL			0
#define ID_TYPE_TSG			1
#define ID_TYPE_UNKNOWN			((u32)~0)

#define PREEMPT_TIMEOUT_RC		1
#define PREEMPT_TIMEOUT_NORC		0

#define GRFIFO_TIMEOUT_CHECK_PERIOD_US 100000

#define RC_TYPE_NORMAL			0
#define RC_TYPE_MMU_FAULT		1
#define RC_TYPE_PBDMA_FAULT		2
#define RC_TYPE_NO_RC			0xff

#define NVGPU_FIFO_DEFAULT_TIMESLICE_TIMEOUT	128UL
#define NVGPU_FIFO_DEFAULT_TIMESLICE_SCALE	3UL

/*
 * Number of entries in the kickoff latency buffer, used to calculate
 * the profiling and histogram. This number is calculated to be statistically
 * significative on a histogram on a 5% step
 */
#ifdef CONFIG_DEBUG_FS
#define FIFO_PROFILING_ENTRIES	16384
#endif

#define	RUNLIST_DISABLED		0
#define	RUNLIST_ENABLED			1

#define RUNLIST_INFO_MUTEX_LOCKED	1

/* generally corresponds to the "pbdma" engine */

struct fifo_runlist_info_gk20a {
	unsigned long *active_channels;
	unsigned long *active_tsgs;
	/* Each engine has its own SW and HW runlist buffer.*/
	struct nvgpu_mem mem[MAX_RUNLIST_BUFFERS];
	u32  cur_buffer;
	u32  total_entries;
	u32  pbdma_bitmask;      /* pbdmas supported for this runlist*/
	u32  eng_bitmask;        /* engines using this runlist */
	u32  reset_eng_bitmask;  /* engines to be reset during recovery */
	bool stopped;
	bool support_tsg;
	struct nvgpu_mutex mutex; /* protect channel preempt and runlist update */
};

enum {
	ENGINE_GR_GK20A	    = 0,
	ENGINE_GRCE_GK20A    = 1,
	ENGINE_ASYNC_CE_GK20A  = 2,
	ENGINE_INVAL_GK20A
};

struct fifo_pbdma_exception_info_gk20a {
	u32 status_r; /* raw register value from hardware */
	u32 id, next_id;
	u32 chan_status_v; /* raw value from hardware */
	bool id_is_chid, next_id_is_chid;
	bool chsw_in_progress;
};

struct fifo_engine_exception_info_gk20a {
	u32 status_r; /* raw register value from hardware */
	u32 id, next_id;
	u32 ctx_status_v; /* raw value from hardware */
	bool id_is_chid, next_id_is_chid;
	bool faulted, idle, ctxsw_in_progress;
};

struct fifo_engine_info_gk20a {
	u32 engine_id;
	u32 runlist_id;
	u32 intr_mask;
	u32 reset_mask;
	u32 pbdma_id;
	u32 inst_id;
	u32 pri_base;
	u32 fault_id;
	u32 engine_enum;
	struct fifo_pbdma_exception_info_gk20a pbdma_exception_info;
	struct fifo_engine_exception_info_gk20a engine_exception_info;
};

enum {
	PROFILE_IOCTL_ENTRY = 0,
	PROFILE_ENTRY,
	PROFILE_JOB_TRACKING,
	PROFILE_APPEND,
	PROFILE_END,
	PROFILE_IOCTL_EXIT,
	PROFILE_MAX
};

struct fifo_profile_gk20a {
	u64 timestamp[PROFILE_MAX];
};

struct fifo_gk20a {
	struct gk20a *g;
	unsigned int num_channels;
	unsigned int runlist_entry_size;
	unsigned int num_runlist_entries;

	unsigned int num_pbdma;
	u32 *pbdma_map;

	struct fifo_engine_info_gk20a *engine_info;
	u32 max_engines;
	u32 num_engines;
	u32 *active_engines_list;

	struct fifo_runlist_info_gk20a *runlist_info;
	u32 max_runlists;
#ifdef CONFIG_DEBUG_FS
	struct {
		struct fifo_profile_gk20a *data;
		nvgpu_atomic_t get;
		bool enabled;
		u64 *sorted;
		struct nvgpu_ref ref;
		struct nvgpu_mutex lock;
	} profile;
#endif
	struct nvgpu_mem userd;
	u32 userd_entry_size;

	unsigned int used_channels;
	struct channel_gk20a *channel;
	/* zero-kref'd channels here */
	struct nvgpu_list_node free_chs;
	struct nvgpu_mutex free_chs_mutex;
	struct nvgpu_mutex gr_reset_mutex;

	struct tsg_gk20a *tsg;
	struct nvgpu_mutex tsg_inuse_mutex;

	void (*remove_support)(struct fifo_gk20a *);
	bool sw_ready;
	struct {
		/* share info between isrs and non-isr code */
		struct {
			struct nvgpu_mutex mutex;
		} isr;
		struct {
			u32 device_fatal_0;
			u32 channel_fatal_0;
			u32 restartable_0;
		} pbdma;
		struct {

		} engine;


	} intr;

	unsigned long deferred_fault_engines;
	bool deferred_reset_pending;
	struct nvgpu_mutex deferred_reset_mutex;

#ifdef CONFIG_TEGRA_19x_GPU
	struct fifo_t19x t19x;
#endif
	u32 channel_base;
};

static inline const char *gk20a_fifo_interleave_level_name(u32 interleave_level)
{
	switch (interleave_level) {
	case NVGPU_RUNLIST_INTERLEAVE_LEVEL_LOW:
		return "LOW";

	case NVGPU_RUNLIST_INTERLEAVE_LEVEL_MEDIUM:
		return "MEDIUM";

	case NVGPU_RUNLIST_INTERLEAVE_LEVEL_HIGH:
		return "HIGH";

	default:
		return "?";
	}
}

struct ch_state {
	int pid;
	int refs;
	bool deterministic;
	u32 inst_block[0];
};

int gk20a_init_fifo_support(struct gk20a *g);

int gk20a_init_fifo_setup_hw(struct gk20a *g);

void gk20a_fifo_isr(struct gk20a *g);
int gk20a_fifo_nonstall_isr(struct gk20a *g);

int gk20a_fifo_preempt_channel(struct gk20a *g, u32 chid);
int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid);
int gk20a_fifo_preempt(struct gk20a *g, struct channel_gk20a *ch);

int gk20a_fifo_enable_engine_activity(struct gk20a *g,
			struct fifo_engine_info_gk20a *eng_info);
int gk20a_fifo_enable_all_engine_activity(struct gk20a *g);
int gk20a_fifo_disable_engine_activity(struct gk20a *g,
			struct fifo_engine_info_gk20a *eng_info,
			bool wait_for_idle);
int gk20a_fifo_disable_all_engine_activity(struct gk20a *g,
				bool wait_for_idle);
void gk20a_fifo_enable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg);
void gk20a_fifo_disable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg);

u32 gk20a_fifo_engines_on_ch(struct gk20a *g, u32 chid);

int gk20a_fifo_reschedule_runlist(struct gk20a *g, u32 runlist_id);

int gk20a_fifo_update_runlist(struct gk20a *g, u32 engine_id, u32 chid,
			      bool add, bool wait_for_finish);

int gk20a_fifo_suspend(struct gk20a *g);

bool gk20a_fifo_mmu_fault_pending(struct gk20a *g);

void gk20a_fifo_recover(struct gk20a *g,
			u32 engine_ids, /* if zero, will be queried from HW */
			u32 hw_id, /* if ~0, will be queried from HW */
			bool hw_id_is_tsg, /* ignored if hw_id == ~0 */
			bool id_is_known, bool verbose);
void gk20a_fifo_recover_ch(struct gk20a *g, u32 chid, bool verbose);
void gk20a_fifo_recover_tsg(struct gk20a *g, u32 tsgid, bool verbose);
int gk20a_fifo_force_reset_ch(struct channel_gk20a *ch,
				u32 err_code, bool verbose);
void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id);
int gk20a_init_fifo_reset_enable_hw(struct gk20a *g);
int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch);

void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g,
		unsigned long fault_id);
int gk20a_fifo_wait_engine_idle(struct gk20a *g);
bool gk20a_fifo_is_engine_busy(struct gk20a *g);
u32 gk20a_fifo_engine_interrupt_mask(struct gk20a *g);
u32 gk20a_fifo_get_pbdma_signature(struct gk20a *g);
u32 gk20a_fifo_get_failing_engine_data(struct gk20a *g,
		int *__id, bool *__is_tsg);
void gk20a_fifo_set_ctx_mmu_error_tsg(struct gk20a *g,
		struct tsg_gk20a *tsg);
void gk20a_fifo_abort_tsg(struct gk20a *g, u32 tsgid, bool preempt);
void gk20a_fifo_set_ctx_mmu_error_ch(struct gk20a *g,
		struct channel_gk20a *refch);
bool gk20a_fifo_error_tsg(struct gk20a *g, struct tsg_gk20a *tsg);
bool gk20a_fifo_error_ch(struct gk20a *g, struct channel_gk20a *refch);

struct channel_gk20a *gk20a_fifo_channel_from_chid(struct gk20a *g,
		u32 chid);

void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg);
int gk20a_fifo_set_runlist_interleave(struct gk20a *g,
				u32 id,
				bool is_tsg,
				u32 runlist_id,
				u32 new_level);
int gk20a_fifo_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice);


const char *gk20a_fifo_interleave_level_name(u32 interleave_level);

int gk20a_fifo_engine_enum_from_type(struct gk20a *g, u32 engine_type,
		u32 *inst_id);

u32 gk20a_fifo_get_engine_ids(struct gk20a *g, u32 engine_id[],
				 u32 engine_id_sz, u32 engine_enum);

void gk20a_fifo_delete_runlist(struct fifo_gk20a *f);

struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g,
							 u32 engine_id);

bool gk20a_fifo_is_valid_engine_id(struct gk20a *g, u32 engine_id);

u32 gk20a_fifo_get_gr_engine_id(struct gk20a *g);

int gk20a_fifo_deferred_reset(struct gk20a *g, struct channel_gk20a *ch);

u32 gk20a_fifo_get_all_ce_engine_reset_mask(struct gk20a *g);

u32 gk20a_fifo_get_fast_ce_runlist_id(struct gk20a *g);

u32 gk20a_fifo_get_gr_runlist_id(struct gk20a *g);

bool gk20a_fifo_is_valid_runlist_id(struct gk20a *g, u32 runlist_id);

int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid,
		bool add, bool wait_for_finish);

int gk20a_fifo_init_engine_info(struct fifo_gk20a *f);

void gk20a_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist);
void gk20a_get_ch_runlist_entry(struct channel_gk20a *ch, u32 *runlist);
void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask,
		 u32 runlist_state, int runlist_mutex_state);

u32 gk20a_fifo_userd_gp_get(struct gk20a *g, struct channel_gk20a *c);
void gk20a_fifo_userd_gp_put(struct gk20a *g, struct channel_gk20a *c);
u64 gk20a_fifo_userd_pb_get(struct gk20a *g, struct channel_gk20a *c);

bool gk20a_is_fault_engine_subid_gpc(struct gk20a *g, u32 engine_subid);
#ifdef CONFIG_DEBUG_FS
struct fifo_profile_gk20a *gk20a_fifo_profile_acquire(struct gk20a *g);
void gk20a_fifo_profile_release(struct gk20a *g,
	struct fifo_profile_gk20a *profile);
#endif

void gk20a_dump_channel_status_ramfc(struct gk20a *g,
				     struct gk20a_debug_output *o,
				     u32 chid,
				     struct ch_state *ch_state);
void gk20a_debug_dump_all_channel_status_ramfc(struct gk20a *g,
		 struct gk20a_debug_output *o);
void gk20a_dump_pbdma_status(struct gk20a *g,
				 struct gk20a_debug_output *o);
void gk20a_dump_eng_status(struct gk20a *g,
				 struct gk20a_debug_output *o);
const char *gk20a_decode_ccsr_chan_status(u32 index);
const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index);
void gk20a_fifo_enable_channel(struct channel_gk20a *ch);
void gk20a_fifo_disable_channel(struct channel_gk20a *ch);

bool gk20a_fifo_channel_status_is_next(struct gk20a *g, u32 chid);
bool gk20a_fifo_channel_status_is_ctx_reload(struct gk20a *g, u32 chid);

struct channel_gk20a *gk20a_refch_from_inst_ptr(struct gk20a *g, u64 inst_ptr);
void gk20a_fifo_channel_unbind(struct channel_gk20a *ch_gk20a);

u32 gk20a_fifo_intr_0_error_mask(struct gk20a *g);

int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, unsigned int id_type,
					 unsigned int timeout_rc_type);
int __locked_fifo_preempt(struct gk20a *g, u32 id, bool is_tsg);
void __locked_fifo_preempt_timeout_rc(struct gk20a *g, u32 id,
					 unsigned int id_type);
int gk20a_fifo_setup_ramfc(struct channel_gk20a *c,
			u64 gpfifo_base, u32 gpfifo_entries,
			unsigned long timeout, u32 flags);
int gk20a_fifo_set_priority(struct channel_gk20a *ch, u32 priority);
int gk20a_fifo_set_timeslice(struct channel_gk20a *ch, unsigned int timeslice);
void gk20a_fifo_setup_ramfc_for_privileged_channel(struct channel_gk20a *c);
int gk20a_fifo_alloc_inst(struct gk20a *g, struct channel_gk20a *ch);
void gk20a_fifo_free_inst(struct gk20a *g, struct channel_gk20a *ch);
int gk20a_fifo_setup_userd(struct channel_gk20a *c);
u32 gk20a_fifo_pbdma_acquire_val(u64 timeout);


void gk20a_fifo_handle_runlist_event(struct gk20a *g);
bool gk20a_fifo_should_defer_engine_reset(struct gk20a *g, u32 engine_id,
			u32 engine_subid, bool fake_fault);

void gk20a_fifo_teardown_ch_tsg(struct gk20a *g, u32 __engine_ids,
			u32 hw_id, unsigned int id_type, unsigned int rc_type,
			 struct mmu_fault_info *mmfault);

bool gk20a_fifo_check_tsg_ctxsw_timeout(struct tsg_gk20a *tsg,
			bool *verbose, u32 *ms);
bool gk20a_fifo_handle_sched_error(struct gk20a *g);

void gk20a_fifo_reset_pbdma_method(struct gk20a *g, int pbdma_id,
			 int pbdma_method_index);
unsigned int gk20a_fifo_handle_pbdma_intr_0(struct gk20a *g, u32 pbdma_id,
			u32 pbdma_intr_0, u32 *handled, u32 *error_notifier);
unsigned int gk20a_fifo_handle_pbdma_intr_1(struct gk20a *g, u32 pbdma_id,
			u32 pbdma_intr_1, u32 *handled, u32 *error_notifier);

u32 gk20a_fifo_default_timeslice_us(struct gk20a *g);

#ifdef CONFIG_TEGRA_GK20A_NVHOST
void gk20a_fifo_add_syncpt_wait_cmd(struct gk20a *g,
			struct priv_cmd_entry *cmd, u32 off,
			u32 id, u32 thresh, u64 gpu_va);
u32 gk20a_fifo_get_syncpt_wait_cmd_size(void);
void gk20a_fifo_add_syncpt_incr_cmd(struct gk20a *g,
			bool wfi_cmd, struct priv_cmd_entry *cmd,
			u32 id, u64 gpu_va);
u32 gk20a_fifo_get_syncpt_incr_cmd_size(bool wfi_cmd);
void gk20a_fifo_free_syncpt_buf(struct channel_gk20a *c,
			struct nvgpu_mem *syncpt_buf);
int gk20a_fifo_alloc_syncpt_buf(struct channel_gk20a *c,
			u32 syncpt_id, struct nvgpu_mem *syncpt_buf);
#endif

void gk20a_fifo_get_mmu_fault_info(struct gk20a *g, u32 mmu_fault_id,
				struct mmu_fault_info *mmfault);
#endif /*__GR_GK20A_H__*/