summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/pmu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/pmu')
-rw-r--r--drivers/gpu/nvgpu/common/pmu/pmu.c574
-rw-r--r--drivers/gpu/nvgpu/common/pmu/pmu_debug.c57
-rw-r--r--drivers/gpu/nvgpu/common/pmu/pmu_fw.c2368
-rw-r--r--drivers/gpu/nvgpu/common/pmu/pmu_ipc.c907
-rw-r--r--drivers/gpu/nvgpu/common/pmu/pmu_perfmon.c293
-rw-r--r--drivers/gpu/nvgpu/common/pmu/pmu_pg.c748
6 files changed, 4947 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/pmu/pmu.c b/drivers/gpu/nvgpu/common/pmu/pmu.c
new file mode 100644
index 00000000..3447f40d
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/pmu/pmu.c
@@ -0,0 +1,574 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/pmu.h>
24#include <nvgpu/dma.h>
25#include <nvgpu/log.h>
26#include <nvgpu/pmuif/nvgpu_gpmu_cmdif.h>
27#include <nvgpu/enabled.h>
28#include <nvgpu/barrier.h>
29#include <nvgpu/timers.h>
30#include <nvgpu/bug.h>
31
32#include "gk20a/gk20a.h"
33
34#define PMU_MEM_SCRUBBING_TIMEOUT_MAX 1000
35#define PMU_MEM_SCRUBBING_TIMEOUT_DEFAULT 10
36
37static int nvgpu_pg_init_task(void *arg);
38
39static int pmu_enable_hw(struct nvgpu_pmu *pmu, bool enable)
40{
41 struct gk20a *g = pmu->g;
42 struct nvgpu_timeout timeout;
43 int err = 0;
44
45 nvgpu_log_fn(g, " %s ", g->name);
46
47 if (enable) {
48 /* bring PMU falcon/engine out of reset */
49 g->ops.pmu.reset_engine(g, true);
50
51 if (g->ops.clock_gating.slcg_pmu_load_gating_prod)
52 g->ops.clock_gating.slcg_pmu_load_gating_prod(g,
53 g->slcg_enabled);
54
55 if (g->ops.clock_gating.blcg_pmu_load_gating_prod)
56 g->ops.clock_gating.blcg_pmu_load_gating_prod(g,
57 g->blcg_enabled);
58
59 /* check for PMU IMEM/DMEM scrubbing complete status */
60 nvgpu_timeout_init(g, &timeout,
61 PMU_MEM_SCRUBBING_TIMEOUT_MAX /
62 PMU_MEM_SCRUBBING_TIMEOUT_DEFAULT,
63 NVGPU_TIMER_RETRY_TIMER);
64 do {
65 if (nvgpu_flcn_get_mem_scrubbing_status(pmu->flcn))
66 goto exit;
67
68 nvgpu_udelay(PMU_MEM_SCRUBBING_TIMEOUT_DEFAULT);
69 } while (!nvgpu_timeout_expired(&timeout));
70
71 /* keep PMU falcon/engine in reset
72 * if IMEM/DMEM scrubbing fails
73 */
74 g->ops.pmu.reset_engine(g, false);
75 nvgpu_err(g, "Falcon mem scrubbing timeout");
76 err = -ETIMEDOUT;
77 } else
78 /* keep PMU falcon/engine in reset */
79 g->ops.pmu.reset_engine(g, false);
80
81exit:
82 nvgpu_log_fn(g, "%s Done, status - %d ", g->name, err);
83 return err;
84}
85
86static int pmu_enable(struct nvgpu_pmu *pmu, bool enable)
87{
88 struct gk20a *g = pmu->g;
89 int err = 0;
90
91 nvgpu_log_fn(g, " ");
92
93 if (!enable) {
94 if (!g->ops.pmu.is_engine_in_reset(g)) {
95 pmu_enable_irq(pmu, false);
96 pmu_enable_hw(pmu, false);
97 }
98 } else {
99 err = pmu_enable_hw(pmu, true);
100 if (err)
101 goto exit;
102
103 err = nvgpu_flcn_wait_idle(pmu->flcn);
104 if (err)
105 goto exit;
106
107 pmu_enable_irq(pmu, true);
108 }
109
110exit:
111 nvgpu_log_fn(g, "Done, status - %d ", err);
112 return err;
113}
114
115int nvgpu_pmu_reset(struct gk20a *g)
116{
117 struct nvgpu_pmu *pmu = &g->pmu;
118 int err = 0;
119
120 nvgpu_log_fn(g, " %s ", g->name);
121
122 err = nvgpu_flcn_wait_idle(pmu->flcn);
123 if (err)
124 goto exit;
125
126 err = pmu_enable(pmu, false);
127 if (err)
128 goto exit;
129
130 err = pmu_enable(pmu, true);
131
132exit:
133 nvgpu_log_fn(g, " %s Done, status - %d ", g->name, err);
134 return err;
135}
136
137static int nvgpu_init_task_pg_init(struct gk20a *g)
138{
139 struct nvgpu_pmu *pmu = &g->pmu;
140 char thread_name[64];
141 int err = 0;
142
143 nvgpu_log_fn(g, " ");
144
145 nvgpu_cond_init(&pmu->pg_init.wq);
146
147 snprintf(thread_name, sizeof(thread_name),
148 "nvgpu_pg_init_%s", g->name);
149
150 err = nvgpu_thread_create(&pmu->pg_init.state_task, g,
151 nvgpu_pg_init_task, thread_name);
152 if (err)
153 nvgpu_err(g, "failed to start nvgpu_pg_init thread");
154
155 return err;
156}
157
158static int nvgpu_init_pmu_setup_sw(struct gk20a *g)
159{
160 struct nvgpu_pmu *pmu = &g->pmu;
161 struct mm_gk20a *mm = &g->mm;
162 struct vm_gk20a *vm = mm->pmu.vm;
163 unsigned int i;
164 int err = 0;
165 u8 *ptr;
166
167 nvgpu_log_fn(g, " ");
168
169 /* start with elpg disabled until first enable call */
170 pmu->elpg_refcnt = 0;
171
172 /* Create thread to handle PMU state machine */
173 nvgpu_init_task_pg_init(g);
174
175 if (pmu->sw_ready) {
176 for (i = 0; i < pmu->mutex_cnt; i++) {
177 pmu->mutex[i].id = i;
178 pmu->mutex[i].index = i;
179 }
180 nvgpu_pmu_seq_init(pmu);
181
182 nvgpu_log_fn(g, "skip init");
183 goto skip_init;
184 }
185
186 /* no infoRom script from vbios? */
187
188 /* TBD: sysmon subtask */
189
190 if (IS_ENABLED(CONFIG_TEGRA_GK20A_PERFMON))
191 pmu->perfmon_sampling_enabled = true;
192
193 pmu->mutex_cnt = g->ops.pmu.pmu_mutex_size();
194 pmu->mutex = nvgpu_kzalloc(g, pmu->mutex_cnt *
195 sizeof(struct pmu_mutex));
196 if (!pmu->mutex) {
197 err = -ENOMEM;
198 goto err;
199 }
200
201 for (i = 0; i < pmu->mutex_cnt; i++) {
202 pmu->mutex[i].id = i;
203 pmu->mutex[i].index = i;
204 }
205
206 pmu->seq = nvgpu_kzalloc(g, PMU_MAX_NUM_SEQUENCES *
207 sizeof(struct pmu_sequence));
208 if (!pmu->seq) {
209 err = -ENOMEM;
210 goto err_free_mutex;
211 }
212
213 nvgpu_pmu_seq_init(pmu);
214
215 err = nvgpu_dma_alloc_map_sys(vm, GK20A_PMU_SEQ_BUF_SIZE,
216 &pmu->seq_buf);
217 if (err) {
218 nvgpu_err(g, "failed to allocate memory");
219 goto err_free_seq;
220 }
221
222 ptr = (u8 *)pmu->seq_buf.cpu_va;
223
224 /* TBD: remove this if ZBC save/restore is handled by PMU
225 * end an empty ZBC sequence for now
226 */
227 ptr[0] = 0x16; /* opcode EXIT */
228 ptr[1] = 0; ptr[2] = 1; ptr[3] = 0;
229 ptr[4] = 0; ptr[5] = 0; ptr[6] = 0; ptr[7] = 0;
230
231 pmu->seq_buf.size = GK20A_PMU_SEQ_BUF_SIZE;
232
233 err = nvgpu_dma_alloc_map(vm, GK20A_PMU_TRACE_BUFSIZE,
234 &pmu->trace_buf);
235 if (err) {
236 nvgpu_err(g, "failed to allocate pmu trace buffer\n");
237 goto err_free_seq_buf;
238 }
239
240 pmu->sw_ready = true;
241
242skip_init:
243 nvgpu_log_fn(g, "done");
244 return 0;
245 err_free_seq_buf:
246 nvgpu_dma_unmap_free(vm, &pmu->seq_buf);
247 err_free_seq:
248 nvgpu_kfree(g, pmu->seq);
249 err_free_mutex:
250 nvgpu_kfree(g, pmu->mutex);
251 err:
252 nvgpu_log_fn(g, "fail");
253 return err;
254}
255
256int nvgpu_init_pmu_support(struct gk20a *g)
257{
258 struct nvgpu_pmu *pmu = &g->pmu;
259 u32 err;
260
261 nvgpu_log_fn(g, " ");
262
263 if (pmu->initialized)
264 return 0;
265
266 err = pmu_enable_hw(pmu, true);
267 if (err)
268 return err;
269
270 if (g->support_pmu) {
271 err = nvgpu_init_pmu_setup_sw(g);
272 if (err)
273 return err;
274 err = g->ops.pmu.pmu_setup_hw_and_bootstrap(g);
275 if (err)
276 return err;
277
278 nvgpu_pmu_state_change(g, PMU_STATE_STARTING, false);
279 }
280
281 return err;
282}
283
284int nvgpu_pmu_process_init_msg(struct nvgpu_pmu *pmu,
285 struct pmu_msg *msg)
286{
287 struct gk20a *g = gk20a_from_pmu(pmu);
288 struct pmu_v *pv = &g->ops.pmu_ver;
289 union pmu_init_msg_pmu *init;
290 struct pmu_sha1_gid_data gid_data;
291 u32 i, tail = 0;
292
293 nvgpu_log_fn(g, " ");
294
295 nvgpu_pmu_dbg(g, "init received\n");
296
297 g->ops.pmu.pmu_msgq_tail(pmu, &tail, QUEUE_GET);
298
299 nvgpu_flcn_copy_from_dmem(pmu->flcn, tail,
300 (u8 *)&msg->hdr, PMU_MSG_HDR_SIZE, 0);
301 if (msg->hdr.unit_id != PMU_UNIT_INIT) {
302 nvgpu_err(g, "expecting init msg");
303 return -EINVAL;
304 }
305
306 nvgpu_flcn_copy_from_dmem(pmu->flcn, tail + PMU_MSG_HDR_SIZE,
307 (u8 *)&msg->msg, msg->hdr.size - PMU_MSG_HDR_SIZE, 0);
308
309 if (msg->msg.init.msg_type != PMU_INIT_MSG_TYPE_PMU_INIT) {
310 nvgpu_err(g, "expecting init msg");
311 return -EINVAL;
312 }
313
314 tail += ALIGN(msg->hdr.size, PMU_DMEM_ALIGNMENT);
315 g->ops.pmu.pmu_msgq_tail(pmu, &tail, QUEUE_SET);
316
317 init = pv->get_pmu_msg_pmu_init_msg_ptr(&(msg->msg.init));
318 if (!pmu->gid_info.valid) {
319
320 nvgpu_flcn_copy_from_dmem(pmu->flcn,
321 pv->get_pmu_init_msg_pmu_sw_mg_off(init),
322 (u8 *)&gid_data,
323 sizeof(struct pmu_sha1_gid_data), 0);
324
325 pmu->gid_info.valid =
326 (*(u32 *)gid_data.signature == PMU_SHA1_GID_SIGNATURE);
327
328 if (pmu->gid_info.valid) {
329
330 BUG_ON(sizeof(pmu->gid_info.gid) !=
331 sizeof(gid_data.gid));
332
333 memcpy(pmu->gid_info.gid, gid_data.gid,
334 sizeof(pmu->gid_info.gid));
335 }
336 }
337
338 for (i = 0; i < PMU_QUEUE_COUNT; i++)
339 nvgpu_pmu_queue_init(pmu, i, init);
340
341 if (!nvgpu_alloc_initialized(&pmu->dmem)) {
342 /* Align start and end addresses */
343 u32 start = ALIGN(pv->get_pmu_init_msg_pmu_sw_mg_off(init),
344 PMU_DMEM_ALLOC_ALIGNMENT);
345 u32 end = (pv->get_pmu_init_msg_pmu_sw_mg_off(init) +
346 pv->get_pmu_init_msg_pmu_sw_mg_size(init)) &
347 ~(PMU_DMEM_ALLOC_ALIGNMENT - 1);
348 u32 size = end - start;
349
350 nvgpu_bitmap_allocator_init(g, &pmu->dmem, "gk20a_pmu_dmem",
351 start, size, PMU_DMEM_ALLOC_ALIGNMENT, 0);
352 }
353
354 pmu->pmu_ready = true;
355
356 nvgpu_pmu_state_change(g, PMU_STATE_INIT_RECEIVED, true);
357
358 nvgpu_pmu_dbg(g, "init received end\n");
359
360 return 0;
361}
362
363static void pmu_setup_hw_enable_elpg(struct gk20a *g)
364{
365 struct nvgpu_pmu *pmu = &g->pmu;
366
367 nvgpu_log_fn(g, " ");
368
369 pmu->initialized = true;
370 nvgpu_pmu_state_change(g, PMU_STATE_STARTED, false);
371
372 if (nvgpu_is_enabled(g, NVGPU_PMU_ZBC_SAVE)) {
373 /* Save zbc table after PMU is initialized. */
374 pmu->zbc_ready = true;
375 gk20a_pmu_save_zbc(g, 0xf);
376 }
377
378 if (g->elpg_enabled) {
379 /* Init reg with prod values*/
380 if (g->ops.pmu.pmu_setup_elpg)
381 g->ops.pmu.pmu_setup_elpg(g);
382 nvgpu_pmu_enable_elpg(g);
383 }
384
385 nvgpu_udelay(50);
386
387 /* Enable AELPG */
388 if (g->aelpg_enabled) {
389 nvgpu_aelpg_init(g);
390 nvgpu_aelpg_init_and_enable(g, PMU_AP_CTRL_ID_GRAPHICS);
391 }
392}
393
394void nvgpu_pmu_state_change(struct gk20a *g, u32 pmu_state,
395 bool post_change_event)
396{
397 struct nvgpu_pmu *pmu = &g->pmu;
398
399 nvgpu_pmu_dbg(g, "pmu_state - %d", pmu_state);
400
401 pmu->pmu_state = pmu_state;
402
403 if (post_change_event) {
404 pmu->pg_init.state_change = true;
405 nvgpu_cond_signal(&pmu->pg_init.wq);
406 }
407
408 /* make status visible */
409 nvgpu_smp_mb();
410}
411
412static int nvgpu_pg_init_task(void *arg)
413{
414 struct gk20a *g = (struct gk20a *)arg;
415 struct nvgpu_pmu *pmu = &g->pmu;
416 struct nvgpu_pg_init *pg_init = &pmu->pg_init;
417 u32 pmu_state = 0;
418
419 nvgpu_log_fn(g, "thread start");
420
421 while (true) {
422
423 NVGPU_COND_WAIT_INTERRUPTIBLE(&pg_init->wq,
424 (pg_init->state_change == true), 0);
425
426 pmu->pg_init.state_change = false;
427 pmu_state = NV_ACCESS_ONCE(pmu->pmu_state);
428
429 if (pmu_state == PMU_STATE_EXIT) {
430 nvgpu_pmu_dbg(g, "pmu state exit");
431 break;
432 }
433
434 switch (pmu_state) {
435 case PMU_STATE_INIT_RECEIVED:
436 nvgpu_pmu_dbg(g, "pmu starting");
437 if (g->can_elpg)
438 nvgpu_pmu_init_powergating(g);
439 break;
440 case PMU_STATE_ELPG_BOOTED:
441 nvgpu_pmu_dbg(g, "elpg booted");
442 nvgpu_pmu_init_bind_fecs(g);
443 break;
444 case PMU_STATE_LOADING_PG_BUF:
445 nvgpu_pmu_dbg(g, "loaded pg buf");
446 nvgpu_pmu_setup_hw_load_zbc(g);
447 break;
448 case PMU_STATE_LOADING_ZBC:
449 nvgpu_pmu_dbg(g, "loaded zbc");
450 pmu_setup_hw_enable_elpg(g);
451 nvgpu_pmu_dbg(g, "PMU booted, thread exiting");
452 return 0;
453 default:
454 nvgpu_pmu_dbg(g, "invalid state");
455 break;
456 }
457
458 }
459
460 while (!nvgpu_thread_should_stop(&pg_init->state_task))
461 nvgpu_usleep_range(5000, 5100);
462
463 nvgpu_log_fn(g, "thread exit");
464
465 return 0;
466}
467
468int nvgpu_pmu_destroy(struct gk20a *g)
469{
470 struct nvgpu_pmu *pmu = &g->pmu;
471 struct pmu_pg_stats_data pg_stat_data = { 0 };
472 struct nvgpu_timeout timeout;
473 int i;
474
475 nvgpu_log_fn(g, " ");
476
477 if (!g->support_pmu)
478 return 0;
479
480 /* make sure the pending operations are finished before we continue */
481 if (nvgpu_thread_is_running(&pmu->pg_init.state_task)) {
482
483 /* post PMU_STATE_EXIT to exit PMU state machine loop */
484 nvgpu_pmu_state_change(g, PMU_STATE_EXIT, true);
485
486 /* Make thread stop*/
487 nvgpu_thread_stop(&pmu->pg_init.state_task);
488
489 /* wait to confirm thread stopped */
490 nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER);
491 do {
492 if (!nvgpu_thread_is_running(&pmu->pg_init.state_task))
493 break;
494 nvgpu_udelay(2);
495 } while (!nvgpu_timeout_expired_msg(&timeout,
496 "timeout - waiting PMU state machine thread stop"));
497 }
498
499 nvgpu_pmu_get_pg_stats(g,
500 PMU_PG_ELPG_ENGINE_ID_GRAPHICS, &pg_stat_data);
501
502 if (nvgpu_pmu_disable_elpg(g))
503 nvgpu_err(g, "failed to set disable elpg");
504 pmu->initialized = false;
505
506 /* update the s/w ELPG residency counters */
507 g->pg_ingating_time_us += (u64)pg_stat_data.ingating_time;
508 g->pg_ungating_time_us += (u64)pg_stat_data.ungating_time;
509 g->pg_gating_cnt += pg_stat_data.gating_cnt;
510
511 nvgpu_mutex_acquire(&pmu->isr_mutex);
512 pmu->isr_enabled = false;
513 nvgpu_mutex_release(&pmu->isr_mutex);
514
515 for (i = 0; i < PMU_QUEUE_COUNT; i++)
516 nvgpu_mutex_destroy(&pmu->queue[i].mutex);
517
518 nvgpu_pmu_state_change(g, PMU_STATE_OFF, false);
519 pmu->pmu_ready = false;
520 pmu->perfmon_ready = false;
521 pmu->zbc_ready = false;
522 g->pmu_lsf_pmu_wpr_init_done = false;
523 __nvgpu_set_enabled(g, NVGPU_PMU_FECS_BOOTSTRAP_DONE, false);
524
525 nvgpu_log_fn(g, "done");
526 return 0;
527}
528
529void nvgpu_pmu_surface_describe(struct gk20a *g, struct nvgpu_mem *mem,
530 struct flcn_mem_desc_v0 *fb)
531{
532 fb->address.lo = u64_lo32(mem->gpu_va);
533 fb->address.hi = u64_hi32(mem->gpu_va);
534 fb->params = ((u32)mem->size & 0xFFFFFF);
535 fb->params |= (GK20A_PMU_DMAIDX_VIRT << 24);
536}
537
538int nvgpu_pmu_vidmem_surface_alloc(struct gk20a *g, struct nvgpu_mem *mem,
539 u32 size)
540{
541 struct mm_gk20a *mm = &g->mm;
542 struct vm_gk20a *vm = mm->pmu.vm;
543 int err;
544
545 err = nvgpu_dma_alloc_map_vid(vm, size, mem);
546 if (err) {
547 nvgpu_err(g, "memory allocation failed");
548 return -ENOMEM;
549 }
550
551 return 0;
552}
553
554int nvgpu_pmu_sysmem_surface_alloc(struct gk20a *g, struct nvgpu_mem *mem,
555 u32 size)
556{
557 struct mm_gk20a *mm = &g->mm;
558 struct vm_gk20a *vm = mm->pmu.vm;
559 int err;
560
561 err = nvgpu_dma_alloc_map_sys(vm, size, mem);
562 if (err) {
563 nvgpu_err(g, "failed to allocate memory\n");
564 return -ENOMEM;
565 }
566
567 return 0;
568}
569
570void nvgpu_pmu_surface_free(struct gk20a *g, struct nvgpu_mem *mem)
571{
572 nvgpu_dma_free(g, mem);
573 memset(mem, 0, sizeof(struct nvgpu_mem));
574}
diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_debug.c b/drivers/gpu/nvgpu/common/pmu/pmu_debug.c
new file mode 100644
index 00000000..6ad82ca8
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/pmu/pmu_debug.c
@@ -0,0 +1,57 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/pmu.h>
24#include <nvgpu/log.h>
25#include <nvgpu/timers.h>
26#include <nvgpu/kmem.h>
27#include <nvgpu/dma.h>
28#include <nvgpu/pmuif/nvgpu_gpmu_cmdif.h>
29
30#include "gk20a/gk20a.h"
31
32void nvgpu_pmu_dump_elpg_stats(struct nvgpu_pmu *pmu)
33{
34 struct gk20a *g = pmu->g;
35
36 /* Print PG stats */
37 nvgpu_err(g, "Print PG stats");
38 nvgpu_flcn_print_dmem(pmu->flcn,
39 pmu->stat_dmem_offset[PMU_PG_ELPG_ENGINE_ID_GRAPHICS],
40 sizeof(struct pmu_pg_stats_v2));
41
42 gk20a_pmu_dump_elpg_stats(pmu);
43}
44
45void nvgpu_pmu_dump_falcon_stats(struct nvgpu_pmu *pmu)
46{
47 struct gk20a *g = pmu->g;
48
49 nvgpu_flcn_dump_stats(pmu->flcn);
50 gk20a_pmu_dump_falcon_stats(pmu);
51
52 nvgpu_err(g, "pmu state: %d", pmu->pmu_state);
53 nvgpu_err(g, "elpg state: %d", pmu->elpg_stat);
54
55 /* PMU may crash due to FECS crash. Dump FECS status */
56 gk20a_fecs_dump_falcon_stats(g);
57}
diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_fw.c b/drivers/gpu/nvgpu/common/pmu/pmu_fw.c
new file mode 100644
index 00000000..654fde21
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/pmu/pmu_fw.c
@@ -0,0 +1,2368 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/pmu.h>
24#include <nvgpu/dma.h>
25#include <nvgpu/log.h>
26#include <nvgpu/pmuif/nvgpu_gpmu_cmdif.h>
27#include <nvgpu/firmware.h>
28#include <nvgpu/enabled.h>
29
30#include "gk20a/gk20a.h"
31
32#include "boardobj/boardobj.h"
33#include "boardobj/boardobjgrp.h"
34
35/* PMU NS UCODE IMG */
36#define NVGPU_PMU_NS_UCODE_IMAGE "gpmu_ucode.bin"
37
38/* PMU F/W version */
39#define APP_VERSION_BIGGPU 22836594
40#define APP_VERSION_NC_3 22204331
41#define APP_VERSION_NC_2 20429989
42#define APP_VERSION_NC_1 20313802
43#define APP_VERSION_NC_0 20360931
44#define APP_VERSION_GM206 20652057
45#define APP_VERSION_NV_GPU 21307569
46#define APP_VERSION_NV_GPU_1 21308030
47#define APP_VERSION_GM20B_5 20490253
48#define APP_VERSION_GM20B_4 19008461
49#define APP_VERSION_GM20B_3 18935575
50#define APP_VERSION_GM20B_2 18694072
51#define APP_VERSION_GM20B_1 18547257
52#define APP_VERSION_GM20B 17615280
53#define APP_VERSION_3 18357968
54#define APP_VERSION_2 18542378
55#define APP_VERSION_1 17997577 /*Obsolete this once 18357968 gets in*/
56#define APP_VERSION_0 16856675
57
58/* PMU version specific functions */
59static u32 pmu_perfmon_cntr_sz_v0(struct nvgpu_pmu *pmu)
60{
61 return sizeof(struct pmu_perfmon_counter_v0);
62}
63
64static u32 pmu_perfmon_cntr_sz_v2(struct nvgpu_pmu *pmu)
65{
66 return sizeof(struct pmu_perfmon_counter_v2);
67}
68
69static void *get_perfmon_cntr_ptr_v2(struct nvgpu_pmu *pmu)
70{
71 return (void *)(&pmu->perfmon_counter_v2);
72}
73
74static void *get_perfmon_cntr_ptr_v0(struct nvgpu_pmu *pmu)
75{
76 return (void *)(&pmu->perfmon_counter_v0);
77}
78
79static void set_perfmon_cntr_ut_v2(struct nvgpu_pmu *pmu, u16 ut)
80{
81 pmu->perfmon_counter_v2.upper_threshold = ut;
82}
83
84static void set_perfmon_cntr_ut_v0(struct nvgpu_pmu *pmu, u16 ut)
85{
86 pmu->perfmon_counter_v0.upper_threshold = ut;
87}
88
89static void set_perfmon_cntr_lt_v2(struct nvgpu_pmu *pmu, u16 lt)
90{
91 pmu->perfmon_counter_v2.lower_threshold = lt;
92}
93
94static void set_perfmon_cntr_lt_v0(struct nvgpu_pmu *pmu, u16 lt)
95{
96 pmu->perfmon_counter_v0.lower_threshold = lt;
97}
98
99static void set_perfmon_cntr_valid_v2(struct nvgpu_pmu *pmu, u8 valid)
100{
101 pmu->perfmon_counter_v2.valid = valid;
102}
103
104static void set_perfmon_cntr_valid_v0(struct nvgpu_pmu *pmu, u8 valid)
105{
106 pmu->perfmon_counter_v0.valid = valid;
107}
108
109static void set_perfmon_cntr_index_v2(struct nvgpu_pmu *pmu, u8 index)
110{
111 pmu->perfmon_counter_v2.index = index;
112}
113
114static void set_perfmon_cntr_index_v0(struct nvgpu_pmu *pmu, u8 index)
115{
116 pmu->perfmon_counter_v0.index = index;
117}
118
119static void set_perfmon_cntr_group_id_v2(struct nvgpu_pmu *pmu, u8 gid)
120{
121 pmu->perfmon_counter_v2.group_id = gid;
122}
123
124static void set_perfmon_cntr_group_id_v0(struct nvgpu_pmu *pmu, u8 gid)
125{
126 pmu->perfmon_counter_v0.group_id = gid;
127}
128
129static u32 pmu_cmdline_size_v0(struct nvgpu_pmu *pmu)
130{
131 return sizeof(struct pmu_cmdline_args_v0);
132}
133
134static u32 pmu_cmdline_size_v1(struct nvgpu_pmu *pmu)
135{
136 return sizeof(struct pmu_cmdline_args_v1);
137}
138
139static u32 pmu_cmdline_size_v2(struct nvgpu_pmu *pmu)
140{
141 return sizeof(struct pmu_cmdline_args_v2);
142}
143
144static void set_pmu_cmdline_args_cpufreq_v2(struct nvgpu_pmu *pmu, u32 freq)
145{
146 pmu->args_v2.cpu_freq_hz = freq;
147}
148static void set_pmu_cmdline_args_secure_mode_v2(struct nvgpu_pmu *pmu, u32 val)
149{
150 pmu->args_v2.secure_mode = val;
151}
152
153static void set_pmu_cmdline_args_falctracesize_v2(
154 struct nvgpu_pmu *pmu, u32 size)
155{
156 pmu->args_v2.falc_trace_size = size;
157}
158
159static void set_pmu_cmdline_args_falctracedmabase_v2(struct nvgpu_pmu *pmu)
160{
161 pmu->args_v2.falc_trace_dma_base = ((u32)pmu->trace_buf.gpu_va)/0x100;
162}
163
164static void set_pmu_cmdline_args_falctracedmaidx_v2(
165 struct nvgpu_pmu *pmu, u32 idx)
166{
167 pmu->args_v2.falc_trace_dma_idx = idx;
168}
169
170
171static void set_pmu_cmdline_args_falctracedmabase_v4(struct nvgpu_pmu *pmu)
172{
173 pmu->args_v4.dma_addr.dma_base = ((u32)pmu->trace_buf.gpu_va)/0x100;
174 pmu->args_v4.dma_addr.dma_base1 = 0;
175 pmu->args_v4.dma_addr.dma_offset = 0;
176}
177
178static u32 pmu_cmdline_size_v4(struct nvgpu_pmu *pmu)
179{
180 return sizeof(struct pmu_cmdline_args_v4);
181}
182
183static void set_pmu_cmdline_args_cpufreq_v4(struct nvgpu_pmu *pmu, u32 freq)
184{
185 pmu->args_v4.cpu_freq_hz = freq;
186}
187static void set_pmu_cmdline_args_secure_mode_v4(struct nvgpu_pmu *pmu, u32 val)
188{
189 pmu->args_v4.secure_mode = val;
190}
191
192static void set_pmu_cmdline_args_falctracesize_v4(
193 struct nvgpu_pmu *pmu, u32 size)
194{
195 pmu->args_v4.falc_trace_size = size;
196}
197static void set_pmu_cmdline_args_falctracedmaidx_v4(
198 struct nvgpu_pmu *pmu, u32 idx)
199{
200 pmu->args_v4.falc_trace_dma_idx = idx;
201}
202
203static u32 pmu_cmdline_size_v5(struct nvgpu_pmu *pmu)
204{
205 return sizeof(struct pmu_cmdline_args_v5);
206}
207
208static u32 pmu_cmdline_size_v6(struct nvgpu_pmu *pmu)
209{
210 return sizeof(struct pmu_cmdline_args_v6);
211}
212
213static void set_pmu_cmdline_args_cpufreq_v5(struct nvgpu_pmu *pmu, u32 freq)
214{
215 pmu->args_v5.cpu_freq_hz = 204000000;
216}
217static void set_pmu_cmdline_args_secure_mode_v5(struct nvgpu_pmu *pmu, u32 val)
218{
219 pmu->args_v5.secure_mode = val;
220}
221
222static void set_pmu_cmdline_args_falctracesize_v5(
223 struct nvgpu_pmu *pmu, u32 size)
224{
225 /* set by surface describe */
226}
227
228static void set_pmu_cmdline_args_falctracedmabase_v5(struct nvgpu_pmu *pmu)
229{
230 struct gk20a *g = gk20a_from_pmu(pmu);
231
232 nvgpu_pmu_surface_describe(g, &pmu->trace_buf, &pmu->args_v5.trace_buf);
233}
234
235static void set_pmu_cmdline_args_falctracedmaidx_v5(
236 struct nvgpu_pmu *pmu, u32 idx)
237{
238 /* set by surface describe */
239}
240
241static u32 pmu_cmdline_size_v3(struct nvgpu_pmu *pmu)
242{
243 return sizeof(struct pmu_cmdline_args_v3);
244}
245
246static void set_pmu_cmdline_args_cpufreq_v3(struct nvgpu_pmu *pmu, u32 freq)
247{
248 pmu->args_v3.cpu_freq_hz = freq;
249}
250static void set_pmu_cmdline_args_secure_mode_v3(struct nvgpu_pmu *pmu, u32 val)
251{
252 pmu->args_v3.secure_mode = val;
253}
254
255static void set_pmu_cmdline_args_falctracesize_v3(
256 struct nvgpu_pmu *pmu, u32 size)
257{
258 pmu->args_v3.falc_trace_size = size;
259}
260
261static void set_pmu_cmdline_args_falctracedmabase_v3(struct nvgpu_pmu *pmu)
262{
263 pmu->args_v3.falc_trace_dma_base = ((u32)pmu->trace_buf.gpu_va)/0x100;
264}
265
266static void set_pmu_cmdline_args_falctracedmaidx_v3(
267 struct nvgpu_pmu *pmu, u32 idx)
268{
269 pmu->args_v3.falc_trace_dma_idx = idx;
270}
271
272static void set_pmu_cmdline_args_cpufreq_v1(struct nvgpu_pmu *pmu, u32 freq)
273{
274 pmu->args_v1.cpu_freq_hz = freq;
275}
276static void set_pmu_cmdline_args_secure_mode_v1(struct nvgpu_pmu *pmu, u32 val)
277{
278 pmu->args_v1.secure_mode = val;
279}
280
281static void set_pmu_cmdline_args_falctracesize_v1(
282 struct nvgpu_pmu *pmu, u32 size)
283{
284 pmu->args_v1.falc_trace_size = size;
285}
286
287static void set_pmu_cmdline_args_falctracedmabase_v1(struct nvgpu_pmu *pmu)
288{
289 pmu->args_v1.falc_trace_dma_base = ((u32)pmu->trace_buf.gpu_va)/0x100;
290}
291
292static void set_pmu_cmdline_args_falctracedmaidx_v1(
293 struct nvgpu_pmu *pmu, u32 idx)
294{
295 pmu->args_v1.falc_trace_dma_idx = idx;
296}
297
298static void set_pmu_cmdline_args_cpufreq_v0(struct nvgpu_pmu *pmu, u32 freq)
299{
300 pmu->args_v0.cpu_freq_hz = freq;
301}
302
303static void *get_pmu_cmdline_args_ptr_v4(struct nvgpu_pmu *pmu)
304{
305 return (void *)(&pmu->args_v4);
306}
307
308static void *get_pmu_cmdline_args_ptr_v3(struct nvgpu_pmu *pmu)
309{
310 return (void *)(&pmu->args_v3);
311}
312
313static void *get_pmu_cmdline_args_ptr_v2(struct nvgpu_pmu *pmu)
314{
315 return (void *)(&pmu->args_v2);
316}
317
318static void *get_pmu_cmdline_args_ptr_v5(struct nvgpu_pmu *pmu)
319{
320 return (void *)(&pmu->args_v5);
321}
322static void *get_pmu_cmdline_args_ptr_v1(struct nvgpu_pmu *pmu)
323{
324 return (void *)(&pmu->args_v1);
325}
326
327static void *get_pmu_cmdline_args_ptr_v0(struct nvgpu_pmu *pmu)
328{
329 return (void *)(&pmu->args_v0);
330}
331
332static u32 get_pmu_allocation_size_v3(struct nvgpu_pmu *pmu)
333{
334 return sizeof(struct pmu_allocation_v3);
335}
336
337static u32 get_pmu_allocation_size_v2(struct nvgpu_pmu *pmu)
338{
339 return sizeof(struct pmu_allocation_v2);
340}
341
342static u32 get_pmu_allocation_size_v1(struct nvgpu_pmu *pmu)
343{
344 return sizeof(struct pmu_allocation_v1);
345}
346
347static u32 get_pmu_allocation_size_v0(struct nvgpu_pmu *pmu)
348{
349 return sizeof(struct pmu_allocation_v0);
350}
351
352static void set_pmu_allocation_ptr_v3(struct nvgpu_pmu *pmu,
353 void **pmu_alloc_ptr, void *assign_ptr)
354{
355 struct pmu_allocation_v3 **pmu_a_ptr =
356 (struct pmu_allocation_v3 **)pmu_alloc_ptr;
357
358 *pmu_a_ptr = (struct pmu_allocation_v3 *)assign_ptr;
359}
360
361static void set_pmu_allocation_ptr_v2(struct nvgpu_pmu *pmu,
362 void **pmu_alloc_ptr, void *assign_ptr)
363{
364 struct pmu_allocation_v2 **pmu_a_ptr =
365 (struct pmu_allocation_v2 **)pmu_alloc_ptr;
366
367 *pmu_a_ptr = (struct pmu_allocation_v2 *)assign_ptr;
368}
369
370static void set_pmu_allocation_ptr_v1(struct nvgpu_pmu *pmu,
371 void **pmu_alloc_ptr, void *assign_ptr)
372{
373 struct pmu_allocation_v1 **pmu_a_ptr =
374 (struct pmu_allocation_v1 **)pmu_alloc_ptr;
375
376 *pmu_a_ptr = (struct pmu_allocation_v1 *)assign_ptr;
377}
378
379static void set_pmu_allocation_ptr_v0(struct nvgpu_pmu *pmu,
380 void **pmu_alloc_ptr, void *assign_ptr)
381{
382 struct pmu_allocation_v0 **pmu_a_ptr =
383 (struct pmu_allocation_v0 **)pmu_alloc_ptr;
384
385 *pmu_a_ptr = (struct pmu_allocation_v0 *)assign_ptr;
386}
387
388static void pmu_allocation_set_dmem_size_v3(struct nvgpu_pmu *pmu,
389 void *pmu_alloc_ptr, u16 size)
390{
391 struct pmu_allocation_v3 *pmu_a_ptr =
392 (struct pmu_allocation_v3 *)pmu_alloc_ptr;
393
394 pmu_a_ptr->alloc.dmem.size = size;
395}
396
397static void pmu_allocation_set_dmem_size_v2(struct nvgpu_pmu *pmu,
398 void *pmu_alloc_ptr, u16 size)
399{
400 struct pmu_allocation_v2 *pmu_a_ptr =
401 (struct pmu_allocation_v2 *)pmu_alloc_ptr;
402
403 pmu_a_ptr->alloc.dmem.size = size;
404}
405
406static void pmu_allocation_set_dmem_size_v1(struct nvgpu_pmu *pmu,
407 void *pmu_alloc_ptr, u16 size)
408{
409 struct pmu_allocation_v1 *pmu_a_ptr =
410 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
411
412 pmu_a_ptr->alloc.dmem.size = size;
413}
414
415static void pmu_allocation_set_dmem_size_v0(struct nvgpu_pmu *pmu,
416 void *pmu_alloc_ptr, u16 size)
417{
418 struct pmu_allocation_v0 *pmu_a_ptr =
419 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
420
421 pmu_a_ptr->alloc.dmem.size = size;
422}
423
424static u16 pmu_allocation_get_dmem_size_v3(struct nvgpu_pmu *pmu,
425 void *pmu_alloc_ptr)
426{
427 struct pmu_allocation_v3 *pmu_a_ptr =
428 (struct pmu_allocation_v3 *)pmu_alloc_ptr;
429
430 return pmu_a_ptr->alloc.dmem.size;
431}
432
433static u16 pmu_allocation_get_dmem_size_v2(struct nvgpu_pmu *pmu,
434 void *pmu_alloc_ptr)
435{
436 struct pmu_allocation_v2 *pmu_a_ptr =
437 (struct pmu_allocation_v2 *)pmu_alloc_ptr;
438
439 return pmu_a_ptr->alloc.dmem.size;
440}
441
442static u16 pmu_allocation_get_dmem_size_v1(struct nvgpu_pmu *pmu,
443 void *pmu_alloc_ptr)
444{
445 struct pmu_allocation_v1 *pmu_a_ptr =
446 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
447
448 return pmu_a_ptr->alloc.dmem.size;
449}
450
451static u16 pmu_allocation_get_dmem_size_v0(struct nvgpu_pmu *pmu,
452 void *pmu_alloc_ptr)
453{
454 struct pmu_allocation_v0 *pmu_a_ptr =
455 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
456
457 return pmu_a_ptr->alloc.dmem.size;
458}
459
460static u32 pmu_allocation_get_dmem_offset_v3(struct nvgpu_pmu *pmu,
461 void *pmu_alloc_ptr)
462{
463 struct pmu_allocation_v3 *pmu_a_ptr =
464 (struct pmu_allocation_v3 *)pmu_alloc_ptr;
465
466 return pmu_a_ptr->alloc.dmem.offset;
467}
468
469static u32 pmu_allocation_get_dmem_offset_v2(struct nvgpu_pmu *pmu,
470 void *pmu_alloc_ptr)
471{
472 struct pmu_allocation_v2 *pmu_a_ptr =
473 (struct pmu_allocation_v2 *)pmu_alloc_ptr;
474
475 return pmu_a_ptr->alloc.dmem.offset;
476}
477
478static u32 pmu_allocation_get_dmem_offset_v1(struct nvgpu_pmu *pmu,
479 void *pmu_alloc_ptr)
480{
481 struct pmu_allocation_v1 *pmu_a_ptr =
482 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
483
484 return pmu_a_ptr->alloc.dmem.offset;
485}
486
487static u32 pmu_allocation_get_dmem_offset_v0(struct nvgpu_pmu *pmu,
488 void *pmu_alloc_ptr)
489{
490 struct pmu_allocation_v0 *pmu_a_ptr =
491 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
492
493 return pmu_a_ptr->alloc.dmem.offset;
494}
495
496static u32 *pmu_allocation_get_dmem_offset_addr_v3(struct nvgpu_pmu *pmu,
497 void *pmu_alloc_ptr)
498{
499 struct pmu_allocation_v3 *pmu_a_ptr =
500 (struct pmu_allocation_v3 *)pmu_alloc_ptr;
501
502 return &pmu_a_ptr->alloc.dmem.offset;
503}
504
505static void *pmu_allocation_get_fb_addr_v3(
506 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr)
507{
508 struct pmu_allocation_v3 *pmu_a_ptr =
509 (struct pmu_allocation_v3 *)pmu_alloc_ptr;
510
511 return (void *)&pmu_a_ptr->alloc.fb;
512}
513
514static u32 pmu_allocation_get_fb_size_v3(
515 struct nvgpu_pmu *pmu, void *pmu_alloc_ptr)
516{
517 struct pmu_allocation_v3 *pmu_a_ptr =
518 (struct pmu_allocation_v3 *)pmu_alloc_ptr;
519
520 return sizeof(pmu_a_ptr->alloc.fb);
521}
522
523static u32 *pmu_allocation_get_dmem_offset_addr_v2(struct nvgpu_pmu *pmu,
524 void *pmu_alloc_ptr)
525{
526 struct pmu_allocation_v2 *pmu_a_ptr =
527 (struct pmu_allocation_v2 *)pmu_alloc_ptr;
528
529 return &pmu_a_ptr->alloc.dmem.offset;
530}
531
532static u32 *pmu_allocation_get_dmem_offset_addr_v1(struct nvgpu_pmu *pmu,
533 void *pmu_alloc_ptr)
534{
535 struct pmu_allocation_v1 *pmu_a_ptr =
536 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
537
538 return &pmu_a_ptr->alloc.dmem.offset;
539}
540
541static u32 *pmu_allocation_get_dmem_offset_addr_v0(struct nvgpu_pmu *pmu,
542 void *pmu_alloc_ptr)
543{
544 struct pmu_allocation_v0 *pmu_a_ptr =
545 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
546
547 return &pmu_a_ptr->alloc.dmem.offset;
548}
549
550static void pmu_allocation_set_dmem_offset_v3(struct nvgpu_pmu *pmu,
551 void *pmu_alloc_ptr, u32 offset)
552{
553 struct pmu_allocation_v3 *pmu_a_ptr =
554 (struct pmu_allocation_v3 *)pmu_alloc_ptr;
555
556 pmu_a_ptr->alloc.dmem.offset = offset;
557}
558
559static void pmu_allocation_set_dmem_offset_v2(struct nvgpu_pmu *pmu,
560 void *pmu_alloc_ptr, u32 offset)
561{
562 struct pmu_allocation_v2 *pmu_a_ptr =
563 (struct pmu_allocation_v2 *)pmu_alloc_ptr;
564
565 pmu_a_ptr->alloc.dmem.offset = offset;
566}
567
568static void pmu_allocation_set_dmem_offset_v1(struct nvgpu_pmu *pmu,
569 void *pmu_alloc_ptr, u32 offset)
570{
571 struct pmu_allocation_v1 *pmu_a_ptr =
572 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
573
574 pmu_a_ptr->alloc.dmem.offset = offset;
575}
576
577static void pmu_allocation_set_dmem_offset_v0(struct nvgpu_pmu *pmu,
578 void *pmu_alloc_ptr, u32 offset)
579{
580 struct pmu_allocation_v0 *pmu_a_ptr =
581 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
582
583 pmu_a_ptr->alloc.dmem.offset = offset;
584}
585
586static void *get_pmu_msg_pmu_init_msg_ptr_v4(struct pmu_init_msg *init)
587{
588 return (void *)(&(init->pmu_init_v4));
589}
590
591static void *get_pmu_msg_pmu_init_msg_ptr_v3(struct pmu_init_msg *init)
592{
593 return (void *)(&(init->pmu_init_v3));
594}
595
596static u16 get_pmu_init_msg_pmu_sw_mg_off_v4(union pmu_init_msg_pmu *init_msg)
597{
598 struct pmu_init_msg_pmu_v4 *init =
599 (struct pmu_init_msg_pmu_v4 *)(&init_msg->v4);
600
601 return init->sw_managed_area_offset;
602}
603
604static u16 get_pmu_init_msg_pmu_sw_mg_off_v3(union pmu_init_msg_pmu *init_msg)
605{
606 struct pmu_init_msg_pmu_v3 *init =
607 (struct pmu_init_msg_pmu_v3 *)(&init_msg->v3);
608
609 return init->sw_managed_area_offset;
610}
611
612static u16 get_pmu_init_msg_pmu_sw_mg_size_v4(union pmu_init_msg_pmu *init_msg)
613{
614 struct pmu_init_msg_pmu_v4 *init =
615 (struct pmu_init_msg_pmu_v4 *)(&init_msg->v4);
616
617 return init->sw_managed_area_size;
618}
619
620static u16 get_pmu_init_msg_pmu_sw_mg_size_v3(union pmu_init_msg_pmu *init_msg)
621{
622 struct pmu_init_msg_pmu_v3 *init =
623 (struct pmu_init_msg_pmu_v3 *)(&init_msg->v3);
624
625 return init->sw_managed_area_size;
626}
627
628static void *get_pmu_msg_pmu_init_msg_ptr_v2(struct pmu_init_msg *init)
629{
630 return (void *)(&(init->pmu_init_v2));
631}
632
633static u16 get_pmu_init_msg_pmu_sw_mg_off_v2(union pmu_init_msg_pmu *init_msg)
634{
635 struct pmu_init_msg_pmu_v2 *init =
636 (struct pmu_init_msg_pmu_v2 *)(&init_msg->v1);
637
638 return init->sw_managed_area_offset;
639}
640
641static u16 get_pmu_init_msg_pmu_sw_mg_size_v2(union pmu_init_msg_pmu *init_msg)
642{
643 struct pmu_init_msg_pmu_v2 *init =
644 (struct pmu_init_msg_pmu_v2 *)(&init_msg->v1);
645
646 return init->sw_managed_area_size;
647}
648
649static void *get_pmu_msg_pmu_init_msg_ptr_v1(struct pmu_init_msg *init)
650{
651 return (void *)(&(init->pmu_init_v1));
652}
653
654static u16 get_pmu_init_msg_pmu_sw_mg_off_v1(union pmu_init_msg_pmu *init_msg)
655{
656 struct pmu_init_msg_pmu_v1 *init =
657 (struct pmu_init_msg_pmu_v1 *)(&init_msg->v1);
658
659 return init->sw_managed_area_offset;
660}
661
662static u16 get_pmu_init_msg_pmu_sw_mg_size_v1(union pmu_init_msg_pmu *init_msg)
663{
664 struct pmu_init_msg_pmu_v1 *init =
665 (struct pmu_init_msg_pmu_v1 *)(&init_msg->v1);
666
667 return init->sw_managed_area_size;
668}
669
670static void *get_pmu_msg_pmu_init_msg_ptr_v0(struct pmu_init_msg *init)
671{
672 return (void *)(&(init->pmu_init_v0));
673}
674
675static u16 get_pmu_init_msg_pmu_sw_mg_off_v0(union pmu_init_msg_pmu *init_msg)
676{
677 struct pmu_init_msg_pmu_v0 *init =
678 (struct pmu_init_msg_pmu_v0 *)(&init_msg->v0);
679
680 return init->sw_managed_area_offset;
681}
682
683static u16 get_pmu_init_msg_pmu_sw_mg_size_v0(union pmu_init_msg_pmu *init_msg)
684{
685 struct pmu_init_msg_pmu_v0 *init =
686 (struct pmu_init_msg_pmu_v0 *)(&init_msg->v0);
687
688 return init->sw_managed_area_size;
689}
690
691static u32 get_pmu_perfmon_cmd_start_size_v3(void)
692{
693 return sizeof(struct pmu_perfmon_cmd_start_v3);
694}
695
696static u32 get_pmu_perfmon_cmd_start_size_v2(void)
697{
698 return sizeof(struct pmu_perfmon_cmd_start_v2);
699}
700
701static u32 get_pmu_perfmon_cmd_start_size_v1(void)
702{
703 return sizeof(struct pmu_perfmon_cmd_start_v1);
704}
705
706static u32 get_pmu_perfmon_cmd_start_size_v0(void)
707{
708 return sizeof(struct pmu_perfmon_cmd_start_v0);
709}
710
711static int get_perfmon_cmd_start_offsetofvar_v3(
712 enum pmu_perfmon_cmd_start_fields field)
713{
714 switch (field) {
715 case COUNTER_ALLOC:
716 return offsetof(struct pmu_perfmon_cmd_start_v3,
717 counter_alloc);
718 default:
719 return -EINVAL;
720 }
721
722 return 0;
723}
724
725static int get_perfmon_cmd_start_offsetofvar_v2(
726 enum pmu_perfmon_cmd_start_fields field)
727{
728 switch (field) {
729 case COUNTER_ALLOC:
730 return offsetof(struct pmu_perfmon_cmd_start_v2,
731 counter_alloc);
732 default:
733 return -EINVAL;
734 }
735
736 return 0;
737}
738
739static int get_perfmon_cmd_start_offsetofvar_v1(
740 enum pmu_perfmon_cmd_start_fields field)
741{
742 switch (field) {
743 case COUNTER_ALLOC:
744 return offsetof(struct pmu_perfmon_cmd_start_v1,
745 counter_alloc);
746 default:
747 return -EINVAL;
748 }
749
750 return 0;
751}
752
753static int get_perfmon_cmd_start_offsetofvar_v0(
754 enum pmu_perfmon_cmd_start_fields field)
755{
756 switch (field) {
757 case COUNTER_ALLOC:
758 return offsetof(struct pmu_perfmon_cmd_start_v0,
759 counter_alloc);
760 default:
761 return -EINVAL;
762 }
763
764 return 0;
765}
766
767static u32 get_pmu_perfmon_cmd_init_size_v3(void)
768{
769 return sizeof(struct pmu_perfmon_cmd_init_v3);
770}
771
772static u32 get_pmu_perfmon_cmd_init_size_v2(void)
773{
774 return sizeof(struct pmu_perfmon_cmd_init_v2);
775}
776
777static u32 get_pmu_perfmon_cmd_init_size_v1(void)
778{
779 return sizeof(struct pmu_perfmon_cmd_init_v1);
780}
781
782static u32 get_pmu_perfmon_cmd_init_size_v0(void)
783{
784 return sizeof(struct pmu_perfmon_cmd_init_v0);
785}
786
787static int get_perfmon_cmd_init_offsetofvar_v3(
788 enum pmu_perfmon_cmd_start_fields field)
789{
790 switch (field) {
791 case COUNTER_ALLOC:
792 return offsetof(struct pmu_perfmon_cmd_init_v3,
793 counter_alloc);
794 default:
795 return -EINVAL;
796 }
797
798 return 0;
799}
800
801static int get_perfmon_cmd_init_offsetofvar_v2(
802 enum pmu_perfmon_cmd_start_fields field)
803{
804 switch (field) {
805 case COUNTER_ALLOC:
806 return offsetof(struct pmu_perfmon_cmd_init_v2,
807 counter_alloc);
808 default:
809 return -EINVAL;
810 }
811
812 return 0;
813}
814
815static int get_perfmon_cmd_init_offsetofvar_v1(
816 enum pmu_perfmon_cmd_start_fields field)
817{
818 switch (field) {
819 case COUNTER_ALLOC:
820 return offsetof(struct pmu_perfmon_cmd_init_v1,
821 counter_alloc);
822 default:
823 return -EINVAL;
824 }
825
826 return 0;
827}
828
829static int get_perfmon_cmd_init_offsetofvar_v0(
830 enum pmu_perfmon_cmd_start_fields field)
831{
832 switch (field) {
833 case COUNTER_ALLOC:
834 return offsetof(struct pmu_perfmon_cmd_init_v0,
835 counter_alloc);
836 default:
837 return -EINVAL;
838 }
839
840 return 0;
841}
842
843static void perfmon_start_set_cmd_type_v3(struct pmu_perfmon_cmd *pc, u8 value)
844{
845 struct pmu_perfmon_cmd_start_v3 *start = &pc->start_v3;
846
847 start->cmd_type = value;
848}
849
850static void perfmon_start_set_cmd_type_v2(struct pmu_perfmon_cmd *pc, u8 value)
851{
852 struct pmu_perfmon_cmd_start_v2 *start = &pc->start_v2;
853
854 start->cmd_type = value;
855}
856
857static void perfmon_start_set_cmd_type_v1(struct pmu_perfmon_cmd *pc, u8 value)
858{
859 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
860
861 start->cmd_type = value;
862}
863
864static void perfmon_start_set_cmd_type_v0(struct pmu_perfmon_cmd *pc, u8 value)
865{
866 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
867
868 start->cmd_type = value;
869}
870
871static void perfmon_start_set_group_id_v3(struct pmu_perfmon_cmd *pc, u8 value)
872{
873 struct pmu_perfmon_cmd_start_v3 *start = &pc->start_v3;
874
875 start->group_id = value;
876}
877
878static void perfmon_start_set_group_id_v2(struct pmu_perfmon_cmd *pc, u8 value)
879{
880 struct pmu_perfmon_cmd_start_v2 *start = &pc->start_v2;
881
882 start->group_id = value;
883}
884
885static void perfmon_start_set_group_id_v1(struct pmu_perfmon_cmd *pc, u8 value)
886{
887 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
888
889 start->group_id = value;
890}
891
892static void perfmon_start_set_group_id_v0(struct pmu_perfmon_cmd *pc, u8 value)
893{
894 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
895
896 start->group_id = value;
897}
898
899static void perfmon_start_set_state_id_v3(struct pmu_perfmon_cmd *pc, u8 value)
900{
901 struct pmu_perfmon_cmd_start_v3 *start = &pc->start_v3;
902
903 start->state_id = value;
904}
905
906static void perfmon_start_set_state_id_v2(struct pmu_perfmon_cmd *pc, u8 value)
907{
908 struct pmu_perfmon_cmd_start_v2 *start = &pc->start_v2;
909
910 start->state_id = value;
911}
912
913static void perfmon_start_set_state_id_v1(struct pmu_perfmon_cmd *pc, u8 value)
914{
915 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
916
917 start->state_id = value;
918}
919
920static void perfmon_start_set_state_id_v0(struct pmu_perfmon_cmd *pc, u8 value)
921{
922 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
923
924 start->state_id = value;
925}
926
927static void perfmon_start_set_flags_v3(struct pmu_perfmon_cmd *pc, u8 value)
928{
929 struct pmu_perfmon_cmd_start_v3 *start = &pc->start_v3;
930
931 start->flags = value;
932}
933
934static void perfmon_start_set_flags_v2(struct pmu_perfmon_cmd *pc, u8 value)
935{
936 struct pmu_perfmon_cmd_start_v2 *start = &pc->start_v2;
937
938 start->flags = value;
939}
940
941static void perfmon_start_set_flags_v1(struct pmu_perfmon_cmd *pc, u8 value)
942{
943 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
944
945 start->flags = value;
946}
947
948static void perfmon_start_set_flags_v0(struct pmu_perfmon_cmd *pc, u8 value)
949{
950 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
951
952 start->flags = value;
953}
954
955static u8 perfmon_start_get_flags_v3(struct pmu_perfmon_cmd *pc)
956{
957 struct pmu_perfmon_cmd_start_v3 *start = &pc->start_v3;
958
959 return start->flags;
960}
961
962static u8 perfmon_start_get_flags_v2(struct pmu_perfmon_cmd *pc)
963{
964 struct pmu_perfmon_cmd_start_v2 *start = &pc->start_v2;
965
966 return start->flags;
967}
968
969static u8 perfmon_start_get_flags_v1(struct pmu_perfmon_cmd *pc)
970{
971 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
972
973 return start->flags;
974}
975
976static u8 perfmon_start_get_flags_v0(struct pmu_perfmon_cmd *pc)
977{
978 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
979
980 return start->flags;
981}
982
983static void perfmon_cmd_init_set_sample_buffer_v3(struct pmu_perfmon_cmd *pc,
984 u16 value)
985{
986 struct pmu_perfmon_cmd_init_v3 *init = &pc->init_v3;
987
988 init->sample_buffer = value;
989}
990
991static void perfmon_cmd_init_set_sample_buffer_v2(struct pmu_perfmon_cmd *pc,
992 u16 value)
993{
994 struct pmu_perfmon_cmd_init_v2 *init = &pc->init_v2;
995
996 init->sample_buffer = value;
997}
998
999
1000static void perfmon_cmd_init_set_sample_buffer_v1(struct pmu_perfmon_cmd *pc,
1001 u16 value)
1002{
1003 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
1004
1005 init->sample_buffer = value;
1006}
1007
1008static void perfmon_cmd_init_set_sample_buffer_v0(struct pmu_perfmon_cmd *pc,
1009 u16 value)
1010{
1011 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
1012
1013 init->sample_buffer = value;
1014}
1015
1016static void perfmon_cmd_init_set_dec_cnt_v3(struct pmu_perfmon_cmd *pc,
1017 u8 value)
1018{
1019 struct pmu_perfmon_cmd_init_v3 *init = &pc->init_v3;
1020
1021 init->to_decrease_count = value;
1022}
1023
1024static void perfmon_cmd_init_set_dec_cnt_v2(struct pmu_perfmon_cmd *pc,
1025 u8 value)
1026{
1027 struct pmu_perfmon_cmd_init_v2 *init = &pc->init_v2;
1028
1029 init->to_decrease_count = value;
1030}
1031
1032static void perfmon_cmd_init_set_dec_cnt_v1(struct pmu_perfmon_cmd *pc,
1033 u8 value)
1034{
1035 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
1036
1037 init->to_decrease_count = value;
1038}
1039
1040static void perfmon_cmd_init_set_dec_cnt_v0(struct pmu_perfmon_cmd *pc,
1041 u8 value)
1042{
1043 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
1044
1045 init->to_decrease_count = value;
1046}
1047
1048static void perfmon_cmd_init_set_base_cnt_id_v3(struct pmu_perfmon_cmd *pc,
1049 u8 value)
1050{
1051 struct pmu_perfmon_cmd_init_v3 *init = &pc->init_v3;
1052
1053 init->base_counter_id = value;
1054}
1055
1056static void perfmon_cmd_init_set_base_cnt_id_v2(struct pmu_perfmon_cmd *pc,
1057 u8 value)
1058{
1059 struct pmu_perfmon_cmd_init_v2 *init = &pc->init_v2;
1060
1061 init->base_counter_id = value;
1062}
1063
1064static void perfmon_cmd_init_set_base_cnt_id_v1(struct pmu_perfmon_cmd *pc,
1065 u8 value)
1066{
1067 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
1068
1069 init->base_counter_id = value;
1070}
1071
1072static void perfmon_cmd_init_set_base_cnt_id_v0(struct pmu_perfmon_cmd *pc,
1073 u8 value)
1074{
1075 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
1076
1077 init->base_counter_id = value;
1078}
1079
1080static void perfmon_cmd_init_set_samp_period_us_v3(struct pmu_perfmon_cmd *pc,
1081 u32 value)
1082{
1083 struct pmu_perfmon_cmd_init_v3 *init = &pc->init_v3;
1084
1085 init->sample_period_us = value;
1086}
1087
1088static void perfmon_cmd_init_set_samp_period_us_v2(struct pmu_perfmon_cmd *pc,
1089 u32 value)
1090{
1091 struct pmu_perfmon_cmd_init_v2 *init = &pc->init_v2;
1092
1093 init->sample_period_us = value;
1094}
1095
1096static void perfmon_cmd_init_set_samp_period_us_v1(struct pmu_perfmon_cmd *pc,
1097 u32 value)
1098{
1099 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
1100
1101 init->sample_period_us = value;
1102}
1103
1104static void perfmon_cmd_init_set_samp_period_us_v0(struct pmu_perfmon_cmd *pc,
1105 u32 value)
1106{
1107 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
1108
1109 init->sample_period_us = value;
1110}
1111
1112static void perfmon_cmd_init_set_num_cnt_v3(struct pmu_perfmon_cmd *pc,
1113 u8 value)
1114{
1115 struct pmu_perfmon_cmd_init_v3 *init = &pc->init_v3;
1116
1117 init->num_counters = value;
1118}
1119
1120static void perfmon_cmd_init_set_num_cnt_v2(struct pmu_perfmon_cmd *pc,
1121 u8 value)
1122{
1123 struct pmu_perfmon_cmd_init_v2 *init = &pc->init_v2;
1124
1125 init->num_counters = value;
1126}
1127
1128static void perfmon_cmd_init_set_num_cnt_v1(struct pmu_perfmon_cmd *pc,
1129 u8 value)
1130{
1131 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
1132
1133 init->num_counters = value;
1134}
1135
1136static void perfmon_cmd_init_set_num_cnt_v0(struct pmu_perfmon_cmd *pc,
1137 u8 value)
1138{
1139 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
1140
1141 init->num_counters = value;
1142}
1143
1144static void perfmon_cmd_init_set_mov_avg_v3(struct pmu_perfmon_cmd *pc,
1145 u8 value)
1146{
1147 struct pmu_perfmon_cmd_init_v3 *init = &pc->init_v3;
1148
1149 init->samples_in_moving_avg = value;
1150}
1151
1152static void perfmon_cmd_init_set_mov_avg_v2(struct pmu_perfmon_cmd *pc,
1153 u8 value)
1154{
1155 struct pmu_perfmon_cmd_init_v2 *init = &pc->init_v2;
1156
1157 init->samples_in_moving_avg = value;
1158}
1159
1160static void perfmon_cmd_init_set_mov_avg_v1(struct pmu_perfmon_cmd *pc,
1161 u8 value)
1162{
1163 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
1164
1165 init->samples_in_moving_avg = value;
1166}
1167
1168static void perfmon_cmd_init_set_mov_avg_v0(struct pmu_perfmon_cmd *pc,
1169 u8 value)
1170{
1171 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
1172
1173 init->samples_in_moving_avg = value;
1174}
1175
1176static void get_pmu_init_msg_pmu_queue_params_v0(struct pmu_queue *queue,
1177 u32 id, void *pmu_init_msg)
1178{
1179 struct pmu_init_msg_pmu_v0 *init =
1180 (struct pmu_init_msg_pmu_v0 *)pmu_init_msg;
1181
1182 queue->index = init->queue_info[id].index;
1183 queue->offset = init->queue_info[id].offset;
1184 queue->size = init->queue_info[id].size;
1185}
1186
1187static void get_pmu_init_msg_pmu_queue_params_v1(struct pmu_queue *queue,
1188 u32 id, void *pmu_init_msg)
1189{
1190 struct pmu_init_msg_pmu_v1 *init =
1191 (struct pmu_init_msg_pmu_v1 *)pmu_init_msg;
1192
1193 queue->index = init->queue_info[id].index;
1194 queue->offset = init->queue_info[id].offset;
1195 queue->size = init->queue_info[id].size;
1196}
1197
1198static void get_pmu_init_msg_pmu_queue_params_v2(struct pmu_queue *queue,
1199 u32 id, void *pmu_init_msg)
1200{
1201 struct pmu_init_msg_pmu_v2 *init =
1202 (struct pmu_init_msg_pmu_v2 *)pmu_init_msg;
1203
1204 queue->index = init->queue_info[id].index;
1205 queue->offset = init->queue_info[id].offset;
1206 queue->size = init->queue_info[id].size;
1207}
1208
1209static void get_pmu_init_msg_pmu_queue_params_v4(struct pmu_queue *queue,
1210 u32 id, void *pmu_init_msg)
1211{
1212 struct pmu_init_msg_pmu_v4 *init = pmu_init_msg;
1213 u32 current_ptr = 0;
1214 u8 i;
1215 u8 tmp_id = id;
1216
1217 if (tmp_id == PMU_COMMAND_QUEUE_HPQ)
1218 tmp_id = PMU_QUEUE_HPQ_IDX_FOR_V3;
1219 else if (tmp_id == PMU_COMMAND_QUEUE_LPQ)
1220 tmp_id = PMU_QUEUE_LPQ_IDX_FOR_V3;
1221 else if (tmp_id == PMU_MESSAGE_QUEUE)
1222 tmp_id = PMU_QUEUE_MSG_IDX_FOR_V3;
1223 else
1224 return;
1225
1226 queue->index = init->queue_index[tmp_id];
1227 queue->size = init->queue_size[tmp_id];
1228 if (tmp_id != 0) {
1229 for (i = 0 ; i < tmp_id; i++)
1230 current_ptr += init->queue_size[i];
1231 }
1232 queue->offset = init->queue_offset + current_ptr;
1233}
1234
1235static void get_pmu_init_msg_pmu_queue_params_v5(struct pmu_queue *queue,
1236 u32 id, void *pmu_init_msg)
1237{
1238 struct pmu_init_msg_pmu_v4 *init = pmu_init_msg;
1239 u32 current_ptr = 0;
1240 u8 i;
1241 u8 tmp_id = id;
1242
1243 if (tmp_id == PMU_COMMAND_QUEUE_HPQ)
1244 tmp_id = PMU_QUEUE_HPQ_IDX_FOR_V3;
1245 else if (tmp_id == PMU_COMMAND_QUEUE_LPQ)
1246 tmp_id = PMU_QUEUE_LPQ_IDX_FOR_V3;
1247 else if (tmp_id == PMU_MESSAGE_QUEUE)
1248 tmp_id = PMU_QUEUE_MSG_IDX_FOR_V4;
1249 else
1250 return;
1251
1252 queue->index = init->queue_index[tmp_id];
1253 queue->size = init->queue_size[tmp_id];
1254 if (tmp_id != 0) {
1255 for (i = 0 ; i < tmp_id; i++)
1256 current_ptr += init->queue_size[i];
1257 }
1258 queue->offset = init->queue_offset + current_ptr;
1259}
1260
1261static void get_pmu_init_msg_pmu_queue_params_v3(struct pmu_queue *queue,
1262 u32 id, void *pmu_init_msg)
1263{
1264 struct pmu_init_msg_pmu_v3 *init =
1265 (struct pmu_init_msg_pmu_v3 *)pmu_init_msg;
1266 u32 current_ptr = 0;
1267 u8 i;
1268 u8 tmp_id = id;
1269
1270 if (tmp_id == PMU_COMMAND_QUEUE_HPQ)
1271 tmp_id = PMU_QUEUE_HPQ_IDX_FOR_V3;
1272 else if (tmp_id == PMU_COMMAND_QUEUE_LPQ)
1273 tmp_id = PMU_QUEUE_LPQ_IDX_FOR_V3;
1274 else if (tmp_id == PMU_MESSAGE_QUEUE)
1275 tmp_id = PMU_QUEUE_MSG_IDX_FOR_V3;
1276 else
1277 return;
1278 queue->index = init->queue_index[tmp_id];
1279 queue->size = init->queue_size[tmp_id];
1280 if (tmp_id != 0) {
1281 for (i = 0 ; i < tmp_id; i++)
1282 current_ptr += init->queue_size[i];
1283 }
1284 queue->offset = init->queue_offset + current_ptr;
1285}
1286
1287static void *get_pmu_sequence_in_alloc_ptr_v3(struct pmu_sequence *seq)
1288{
1289 return (void *)(&seq->in_v3);
1290}
1291
1292static void *get_pmu_sequence_in_alloc_ptr_v1(struct pmu_sequence *seq)
1293{
1294 return (void *)(&seq->in_v1);
1295}
1296
1297static void *get_pmu_sequence_in_alloc_ptr_v0(struct pmu_sequence *seq)
1298{
1299 return (void *)(&seq->in_v0);
1300}
1301
1302static void *get_pmu_sequence_out_alloc_ptr_v3(struct pmu_sequence *seq)
1303{
1304 return (void *)(&seq->out_v3);
1305}
1306
1307static void *get_pmu_sequence_out_alloc_ptr_v1(struct pmu_sequence *seq)
1308{
1309 return (void *)(&seq->out_v1);
1310}
1311
1312static void *get_pmu_sequence_out_alloc_ptr_v0(struct pmu_sequence *seq)
1313{
1314 return (void *)(&seq->out_v0);
1315}
1316
1317static u8 pg_cmd_eng_buf_load_size_v0(struct pmu_pg_cmd *pg)
1318{
1319 return sizeof(pg->eng_buf_load_v0);
1320}
1321
1322static u8 pg_cmd_eng_buf_load_size_v1(struct pmu_pg_cmd *pg)
1323{
1324 return sizeof(pg->eng_buf_load_v1);
1325}
1326
1327static u8 pg_cmd_eng_buf_load_size_v2(struct pmu_pg_cmd *pg)
1328{
1329 return sizeof(pg->eng_buf_load_v2);
1330}
1331
1332static void pg_cmd_eng_buf_load_set_cmd_type_v0(struct pmu_pg_cmd *pg,
1333 u8 value)
1334{
1335 pg->eng_buf_load_v0.cmd_type = value;
1336}
1337
1338static void pg_cmd_eng_buf_load_set_cmd_type_v1(struct pmu_pg_cmd *pg,
1339 u8 value)
1340{
1341 pg->eng_buf_load_v1.cmd_type = value;
1342}
1343
1344static void pg_cmd_eng_buf_load_set_cmd_type_v2(struct pmu_pg_cmd *pg,
1345 u8 value)
1346{
1347 pg->eng_buf_load_v2.cmd_type = value;
1348}
1349
1350static void pg_cmd_eng_buf_load_set_engine_id_v0(struct pmu_pg_cmd *pg,
1351 u8 value)
1352{
1353 pg->eng_buf_load_v0.engine_id = value;
1354}
1355static void pg_cmd_eng_buf_load_set_engine_id_v1(struct pmu_pg_cmd *pg,
1356 u8 value)
1357{
1358 pg->eng_buf_load_v1.engine_id = value;
1359}
1360static void pg_cmd_eng_buf_load_set_engine_id_v2(struct pmu_pg_cmd *pg,
1361 u8 value)
1362{
1363 pg->eng_buf_load_v2.engine_id = value;
1364}
1365static void pg_cmd_eng_buf_load_set_buf_idx_v0(struct pmu_pg_cmd *pg,
1366 u8 value)
1367{
1368 pg->eng_buf_load_v0.buf_idx = value;
1369}
1370static void pg_cmd_eng_buf_load_set_buf_idx_v1(struct pmu_pg_cmd *pg,
1371 u8 value)
1372{
1373 pg->eng_buf_load_v1.buf_idx = value;
1374}
1375static void pg_cmd_eng_buf_load_set_buf_idx_v2(struct pmu_pg_cmd *pg,
1376 u8 value)
1377{
1378 pg->eng_buf_load_v2.buf_idx = value;
1379}
1380
1381static void pg_cmd_eng_buf_load_set_pad_v0(struct pmu_pg_cmd *pg,
1382 u8 value)
1383{
1384 pg->eng_buf_load_v0.pad = value;
1385}
1386static void pg_cmd_eng_buf_load_set_pad_v1(struct pmu_pg_cmd *pg,
1387 u8 value)
1388{
1389 pg->eng_buf_load_v1.pad = value;
1390}
1391static void pg_cmd_eng_buf_load_set_pad_v2(struct pmu_pg_cmd *pg,
1392 u8 value)
1393{
1394 pg->eng_buf_load_v2.pad = value;
1395}
1396
1397static void pg_cmd_eng_buf_load_set_buf_size_v0(struct pmu_pg_cmd *pg,
1398 u16 value)
1399{
1400 pg->eng_buf_load_v0.buf_size = value;
1401}
1402static void pg_cmd_eng_buf_load_set_buf_size_v1(struct pmu_pg_cmd *pg,
1403 u16 value)
1404{
1405 pg->eng_buf_load_v1.dma_desc.dma_size = value;
1406}
1407static void pg_cmd_eng_buf_load_set_buf_size_v2(struct pmu_pg_cmd *pg,
1408 u16 value)
1409{
1410 pg->eng_buf_load_v2.dma_desc.params = value;
1411}
1412
1413static void pg_cmd_eng_buf_load_set_dma_base_v0(struct pmu_pg_cmd *pg,
1414 u32 value)
1415{
1416 pg->eng_buf_load_v0.dma_base = (value >> 8);
1417}
1418static void pg_cmd_eng_buf_load_set_dma_base_v1(struct pmu_pg_cmd *pg,
1419 u32 value)
1420{
1421 pg->eng_buf_load_v1.dma_desc.dma_addr.lo |= u64_lo32(value);
1422 pg->eng_buf_load_v1.dma_desc.dma_addr.hi |= u64_hi32(value);
1423}
1424static void pg_cmd_eng_buf_load_set_dma_base_v2(struct pmu_pg_cmd *pg,
1425 u32 value)
1426{
1427 pg->eng_buf_load_v2.dma_desc.address.lo = u64_lo32(value);
1428 pg->eng_buf_load_v2.dma_desc.address.hi = u64_lo32(value);
1429}
1430
1431static void pg_cmd_eng_buf_load_set_dma_offset_v0(struct pmu_pg_cmd *pg,
1432 u8 value)
1433{
1434 pg->eng_buf_load_v0.dma_offset = value;
1435}
1436static void pg_cmd_eng_buf_load_set_dma_offset_v1(struct pmu_pg_cmd *pg,
1437 u8 value)
1438{
1439 pg->eng_buf_load_v1.dma_desc.dma_addr.lo |= value;
1440}
1441static void pg_cmd_eng_buf_load_set_dma_offset_v2(struct pmu_pg_cmd *pg,
1442 u8 value)
1443{
1444 pg->eng_buf_load_v2.dma_desc.address.lo |= u64_lo32(value);
1445 pg->eng_buf_load_v2.dma_desc.address.hi |= u64_lo32(value);
1446}
1447
1448static void pg_cmd_eng_buf_load_set_dma_idx_v0(struct pmu_pg_cmd *pg,
1449 u8 value)
1450{
1451 pg->eng_buf_load_v0.dma_idx = value;
1452}
1453
1454static void pg_cmd_eng_buf_load_set_dma_idx_v1(struct pmu_pg_cmd *pg,
1455 u8 value)
1456{
1457 pg->eng_buf_load_v1.dma_desc.dma_idx = value;
1458}
1459
1460static void pg_cmd_eng_buf_load_set_dma_idx_v2(struct pmu_pg_cmd *pg,
1461 u8 value)
1462{
1463 pg->eng_buf_load_v2.dma_desc.params |= (value << 24);
1464}
1465
1466static int nvgpu_init_pmu_fw_ver_ops(struct nvgpu_pmu *pmu)
1467{
1468 struct gk20a *g = gk20a_from_pmu(pmu);
1469 struct pmu_v *pv = &g->ops.pmu_ver;
1470 int err = 0;
1471
1472 nvgpu_log_fn(g, " ");
1473
1474 switch (pmu->desc->app_version) {
1475 case APP_VERSION_NC_2:
1476 case APP_VERSION_NC_1:
1477 case APP_VERSION_NC_0:
1478 g->ops.pmu_ver.pg_cmd_eng_buf_load_size =
1479 pg_cmd_eng_buf_load_size_v1;
1480 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_cmd_type =
1481 pg_cmd_eng_buf_load_set_cmd_type_v1;
1482 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_engine_id =
1483 pg_cmd_eng_buf_load_set_engine_id_v1;
1484 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_idx =
1485 pg_cmd_eng_buf_load_set_buf_idx_v1;
1486 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_pad =
1487 pg_cmd_eng_buf_load_set_pad_v1;
1488 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_size =
1489 pg_cmd_eng_buf_load_set_buf_size_v1;
1490 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_base =
1491 pg_cmd_eng_buf_load_set_dma_base_v1;
1492 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_offset =
1493 pg_cmd_eng_buf_load_set_dma_offset_v1;
1494 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_idx =
1495 pg_cmd_eng_buf_load_set_dma_idx_v1;
1496 g->ops.pmu_ver.get_perfmon_cntr_ptr = get_perfmon_cntr_ptr_v2;
1497 g->ops.pmu_ver.set_perfmon_cntr_ut = set_perfmon_cntr_ut_v2;
1498 g->ops.pmu_ver.set_perfmon_cntr_lt = set_perfmon_cntr_lt_v2;
1499 g->ops.pmu_ver.set_perfmon_cntr_valid =
1500 set_perfmon_cntr_valid_v2;
1501 g->ops.pmu_ver.set_perfmon_cntr_index =
1502 set_perfmon_cntr_index_v2;
1503 g->ops.pmu_ver.set_perfmon_cntr_group_id =
1504 set_perfmon_cntr_group_id_v2;
1505 g->ops.pmu_ver.get_perfmon_cntr_sz = pmu_perfmon_cntr_sz_v2;
1506 g->pmu_ver_cmd_id_zbc_table_update = 16;
1507 __nvgpu_set_enabled(g, NVGPU_PMU_ZBC_SAVE, true);
1508 g->ops.pmu_ver.get_pmu_cmdline_args_size =
1509 pmu_cmdline_size_v4;
1510 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq =
1511 set_pmu_cmdline_args_cpufreq_v4;
1512 g->ops.pmu_ver.set_pmu_cmdline_args_secure_mode =
1513 set_pmu_cmdline_args_secure_mode_v4;
1514 g->ops.pmu_ver.set_pmu_cmdline_args_trace_size =
1515 set_pmu_cmdline_args_falctracesize_v4;
1516 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base =
1517 set_pmu_cmdline_args_falctracedmabase_v4;
1518 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx =
1519 set_pmu_cmdline_args_falctracedmaidx_v4;
1520 g->ops.pmu_ver.get_pmu_cmdline_args_ptr =
1521 get_pmu_cmdline_args_ptr_v4;
1522 g->ops.pmu_ver.get_pmu_allocation_struct_size =
1523 get_pmu_allocation_size_v2;
1524 g->ops.pmu_ver.set_pmu_allocation_ptr =
1525 set_pmu_allocation_ptr_v2;
1526 g->ops.pmu_ver.pmu_allocation_set_dmem_size =
1527 pmu_allocation_set_dmem_size_v2;
1528 g->ops.pmu_ver.pmu_allocation_get_dmem_size =
1529 pmu_allocation_get_dmem_size_v2;
1530 g->ops.pmu_ver.pmu_allocation_get_dmem_offset =
1531 pmu_allocation_get_dmem_offset_v2;
1532 g->ops.pmu_ver.pmu_allocation_get_dmem_offset_addr =
1533 pmu_allocation_get_dmem_offset_addr_v2;
1534 g->ops.pmu_ver.pmu_allocation_set_dmem_offset =
1535 pmu_allocation_set_dmem_offset_v2;
1536 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
1537 get_pmu_init_msg_pmu_queue_params_v1;
1538 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
1539 get_pmu_msg_pmu_init_msg_ptr_v1;
1540 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
1541 get_pmu_init_msg_pmu_sw_mg_off_v1;
1542 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
1543 get_pmu_init_msg_pmu_sw_mg_size_v1;
1544 g->ops.pmu_ver.get_pmu_perfmon_cmd_start_size =
1545 get_pmu_perfmon_cmd_start_size_v2;
1546 g->ops.pmu_ver.get_perfmon_cmd_start_offsetofvar =
1547 get_perfmon_cmd_start_offsetofvar_v2;
1548 g->ops.pmu_ver.perfmon_start_set_cmd_type =
1549 perfmon_start_set_cmd_type_v2;
1550 g->ops.pmu_ver.perfmon_start_set_group_id =
1551 perfmon_start_set_group_id_v2;
1552 g->ops.pmu_ver.perfmon_start_set_state_id =
1553 perfmon_start_set_state_id_v2;
1554 g->ops.pmu_ver.perfmon_start_set_flags =
1555 perfmon_start_set_flags_v2;
1556 g->ops.pmu_ver.perfmon_start_get_flags =
1557 perfmon_start_get_flags_v2;
1558 g->ops.pmu_ver.get_pmu_perfmon_cmd_init_size =
1559 get_pmu_perfmon_cmd_init_size_v2;
1560 g->ops.pmu_ver.get_perfmon_cmd_init_offsetofvar =
1561 get_perfmon_cmd_init_offsetofvar_v2;
1562 g->ops.pmu_ver.perfmon_cmd_init_set_sample_buffer =
1563 perfmon_cmd_init_set_sample_buffer_v2;
1564 g->ops.pmu_ver.perfmon_cmd_init_set_dec_cnt =
1565 perfmon_cmd_init_set_dec_cnt_v2;
1566 g->ops.pmu_ver.perfmon_cmd_init_set_base_cnt_id =
1567 perfmon_cmd_init_set_base_cnt_id_v2;
1568 g->ops.pmu_ver.perfmon_cmd_init_set_samp_period_us =
1569 perfmon_cmd_init_set_samp_period_us_v2;
1570 g->ops.pmu_ver.perfmon_cmd_init_set_num_cnt =
1571 perfmon_cmd_init_set_num_cnt_v2;
1572 g->ops.pmu_ver.perfmon_cmd_init_set_mov_avg =
1573 perfmon_cmd_init_set_mov_avg_v2;
1574 g->ops.pmu_ver.get_pmu_seq_in_a_ptr =
1575 get_pmu_sequence_in_alloc_ptr_v1;
1576 g->ops.pmu_ver.get_pmu_seq_out_a_ptr =
1577 get_pmu_sequence_out_alloc_ptr_v1;
1578 break;
1579 case APP_VERSION_NC_3:
1580 case APP_VERSION_BIGGPU:
1581 g->ops.pmu_ver.pg_cmd_eng_buf_load_size =
1582 pg_cmd_eng_buf_load_size_v2;
1583 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_cmd_type =
1584 pg_cmd_eng_buf_load_set_cmd_type_v2;
1585 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_engine_id =
1586 pg_cmd_eng_buf_load_set_engine_id_v2;
1587 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_idx =
1588 pg_cmd_eng_buf_load_set_buf_idx_v2;
1589 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_pad =
1590 pg_cmd_eng_buf_load_set_pad_v2;
1591 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_size =
1592 pg_cmd_eng_buf_load_set_buf_size_v2;
1593 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_base =
1594 pg_cmd_eng_buf_load_set_dma_base_v2;
1595 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_offset =
1596 pg_cmd_eng_buf_load_set_dma_offset_v2;
1597 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_idx =
1598 pg_cmd_eng_buf_load_set_dma_idx_v2;
1599 g->ops.pmu_ver.get_perfmon_cntr_ptr = get_perfmon_cntr_ptr_v2;
1600 g->ops.pmu_ver.set_perfmon_cntr_ut = set_perfmon_cntr_ut_v2;
1601 g->ops.pmu_ver.set_perfmon_cntr_lt = set_perfmon_cntr_lt_v2;
1602 g->ops.pmu_ver.set_perfmon_cntr_valid =
1603 set_perfmon_cntr_valid_v2;
1604 g->ops.pmu_ver.set_perfmon_cntr_index =
1605 set_perfmon_cntr_index_v2;
1606 g->ops.pmu_ver.set_perfmon_cntr_group_id =
1607 set_perfmon_cntr_group_id_v2;
1608 g->ops.pmu_ver.get_perfmon_cntr_sz = pmu_perfmon_cntr_sz_v2;
1609 g->pmu_ver_cmd_id_zbc_table_update = 16;
1610 __nvgpu_set_enabled(g, NVGPU_PMU_ZBC_SAVE, false);
1611 g->ops.pmu_ver.get_pmu_cmdline_args_size =
1612 pmu_cmdline_size_v6;
1613 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq =
1614 set_pmu_cmdline_args_cpufreq_v5;
1615 g->ops.pmu_ver.set_pmu_cmdline_args_secure_mode =
1616 set_pmu_cmdline_args_secure_mode_v5;
1617 g->ops.pmu_ver.set_pmu_cmdline_args_trace_size =
1618 set_pmu_cmdline_args_falctracesize_v5;
1619 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base =
1620 set_pmu_cmdline_args_falctracedmabase_v5;
1621 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx =
1622 set_pmu_cmdline_args_falctracedmaidx_v5;
1623 g->ops.pmu_ver.get_pmu_cmdline_args_ptr =
1624 get_pmu_cmdline_args_ptr_v5;
1625 g->ops.pmu_ver.get_pmu_allocation_struct_size =
1626 get_pmu_allocation_size_v3;
1627 g->ops.pmu_ver.set_pmu_allocation_ptr =
1628 set_pmu_allocation_ptr_v3;
1629 g->ops.pmu_ver.pmu_allocation_set_dmem_size =
1630 pmu_allocation_set_dmem_size_v3;
1631 g->ops.pmu_ver.pmu_allocation_get_dmem_size =
1632 pmu_allocation_get_dmem_size_v3;
1633 g->ops.pmu_ver.pmu_allocation_get_dmem_offset =
1634 pmu_allocation_get_dmem_offset_v3;
1635 g->ops.pmu_ver.pmu_allocation_get_dmem_offset_addr =
1636 pmu_allocation_get_dmem_offset_addr_v3;
1637 g->ops.pmu_ver.pmu_allocation_set_dmem_offset =
1638 pmu_allocation_set_dmem_offset_v3;
1639 g->ops.pmu_ver.pmu_allocation_get_fb_addr =
1640 pmu_allocation_get_fb_addr_v3;
1641 g->ops.pmu_ver.pmu_allocation_get_fb_size =
1642 pmu_allocation_get_fb_size_v3;
1643 if (pmu->desc->app_version == APP_VERSION_BIGGPU)
1644 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
1645 get_pmu_init_msg_pmu_queue_params_v5;
1646 else
1647 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
1648 get_pmu_init_msg_pmu_queue_params_v4;
1649 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
1650 get_pmu_msg_pmu_init_msg_ptr_v4;
1651 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
1652 get_pmu_init_msg_pmu_sw_mg_off_v4;
1653 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
1654 get_pmu_init_msg_pmu_sw_mg_size_v4;
1655 g->ops.pmu_ver.get_pmu_perfmon_cmd_start_size =
1656 get_pmu_perfmon_cmd_start_size_v3;
1657 g->ops.pmu_ver.get_perfmon_cmd_start_offsetofvar =
1658 get_perfmon_cmd_start_offsetofvar_v3;
1659 g->ops.pmu_ver.perfmon_start_set_cmd_type =
1660 perfmon_start_set_cmd_type_v3;
1661 g->ops.pmu_ver.perfmon_start_set_group_id =
1662 perfmon_start_set_group_id_v3;
1663 g->ops.pmu_ver.perfmon_start_set_state_id =
1664 perfmon_start_set_state_id_v3;
1665 g->ops.pmu_ver.perfmon_start_set_flags =
1666 perfmon_start_set_flags_v3;
1667 g->ops.pmu_ver.perfmon_start_get_flags =
1668 perfmon_start_get_flags_v3;
1669 g->ops.pmu_ver.get_pmu_perfmon_cmd_init_size =
1670 get_pmu_perfmon_cmd_init_size_v3;
1671 g->ops.pmu_ver.get_perfmon_cmd_init_offsetofvar =
1672 get_perfmon_cmd_init_offsetofvar_v3;
1673 g->ops.pmu_ver.perfmon_cmd_init_set_sample_buffer =
1674 perfmon_cmd_init_set_sample_buffer_v3;
1675 g->ops.pmu_ver.perfmon_cmd_init_set_dec_cnt =
1676 perfmon_cmd_init_set_dec_cnt_v3;
1677 g->ops.pmu_ver.perfmon_cmd_init_set_base_cnt_id =
1678 perfmon_cmd_init_set_base_cnt_id_v3;
1679 g->ops.pmu_ver.perfmon_cmd_init_set_samp_period_us =
1680 perfmon_cmd_init_set_samp_period_us_v3;
1681 g->ops.pmu_ver.perfmon_cmd_init_set_num_cnt =
1682 perfmon_cmd_init_set_num_cnt_v3;
1683 g->ops.pmu_ver.perfmon_cmd_init_set_mov_avg =
1684 perfmon_cmd_init_set_mov_avg_v3;
1685 g->ops.pmu_ver.get_pmu_seq_in_a_ptr =
1686 get_pmu_sequence_in_alloc_ptr_v3;
1687 g->ops.pmu_ver.get_pmu_seq_out_a_ptr =
1688 get_pmu_sequence_out_alloc_ptr_v3;
1689 break;
1690 case APP_VERSION_GM206:
1691 case APP_VERSION_NV_GPU:
1692 case APP_VERSION_NV_GPU_1:
1693 g->ops.pmu_ver.pg_cmd_eng_buf_load_size =
1694 pg_cmd_eng_buf_load_size_v2;
1695 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_cmd_type =
1696 pg_cmd_eng_buf_load_set_cmd_type_v2;
1697 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_engine_id =
1698 pg_cmd_eng_buf_load_set_engine_id_v2;
1699 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_idx =
1700 pg_cmd_eng_buf_load_set_buf_idx_v2;
1701 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_pad =
1702 pg_cmd_eng_buf_load_set_pad_v2;
1703 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_size =
1704 pg_cmd_eng_buf_load_set_buf_size_v2;
1705 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_base =
1706 pg_cmd_eng_buf_load_set_dma_base_v2;
1707 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_offset =
1708 pg_cmd_eng_buf_load_set_dma_offset_v2;
1709 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_idx =
1710 pg_cmd_eng_buf_load_set_dma_idx_v2;
1711 g->ops.pmu_ver.get_perfmon_cntr_ptr = get_perfmon_cntr_ptr_v2;
1712 g->ops.pmu_ver.set_perfmon_cntr_ut = set_perfmon_cntr_ut_v2;
1713 g->ops.pmu_ver.set_perfmon_cntr_lt = set_perfmon_cntr_lt_v2;
1714 g->ops.pmu_ver.set_perfmon_cntr_valid =
1715 set_perfmon_cntr_valid_v2;
1716 g->ops.pmu_ver.set_perfmon_cntr_index =
1717 set_perfmon_cntr_index_v2;
1718 g->ops.pmu_ver.set_perfmon_cntr_group_id =
1719 set_perfmon_cntr_group_id_v2;
1720 g->ops.pmu_ver.get_perfmon_cntr_sz = pmu_perfmon_cntr_sz_v2;
1721 g->pmu_ver_cmd_id_zbc_table_update = 16;
1722 __nvgpu_set_enabled(g, NVGPU_PMU_ZBC_SAVE, true);
1723 g->ops.pmu_ver.get_pmu_cmdline_args_size =
1724 pmu_cmdline_size_v5;
1725 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq =
1726 set_pmu_cmdline_args_cpufreq_v5;
1727 g->ops.pmu_ver.set_pmu_cmdline_args_secure_mode =
1728 set_pmu_cmdline_args_secure_mode_v5;
1729 g->ops.pmu_ver.set_pmu_cmdline_args_trace_size =
1730 set_pmu_cmdline_args_falctracesize_v5;
1731 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base =
1732 set_pmu_cmdline_args_falctracedmabase_v5;
1733 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx =
1734 set_pmu_cmdline_args_falctracedmaidx_v5;
1735 g->ops.pmu_ver.get_pmu_cmdline_args_ptr =
1736 get_pmu_cmdline_args_ptr_v5;
1737 g->ops.pmu_ver.get_pmu_allocation_struct_size =
1738 get_pmu_allocation_size_v3;
1739 g->ops.pmu_ver.set_pmu_allocation_ptr =
1740 set_pmu_allocation_ptr_v3;
1741 g->ops.pmu_ver.pmu_allocation_set_dmem_size =
1742 pmu_allocation_set_dmem_size_v3;
1743 g->ops.pmu_ver.pmu_allocation_get_dmem_size =
1744 pmu_allocation_get_dmem_size_v3;
1745 g->ops.pmu_ver.pmu_allocation_get_dmem_offset =
1746 pmu_allocation_get_dmem_offset_v3;
1747 g->ops.pmu_ver.pmu_allocation_get_dmem_offset_addr =
1748 pmu_allocation_get_dmem_offset_addr_v3;
1749 g->ops.pmu_ver.pmu_allocation_set_dmem_offset =
1750 pmu_allocation_set_dmem_offset_v3;
1751 g->ops.pmu_ver.pmu_allocation_get_fb_addr =
1752 pmu_allocation_get_fb_addr_v3;
1753 g->ops.pmu_ver.pmu_allocation_get_fb_size =
1754 pmu_allocation_get_fb_size_v3;
1755 if (pmu->desc->app_version != APP_VERSION_NV_GPU &&
1756 pmu->desc->app_version != APP_VERSION_NV_GPU_1) {
1757 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
1758 get_pmu_init_msg_pmu_queue_params_v2;
1759 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
1760 get_pmu_msg_pmu_init_msg_ptr_v2;
1761 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
1762 get_pmu_init_msg_pmu_sw_mg_off_v2;
1763 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
1764 get_pmu_init_msg_pmu_sw_mg_size_v2;
1765 } else {
1766 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
1767 get_pmu_init_msg_pmu_queue_params_v3;
1768 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
1769 get_pmu_msg_pmu_init_msg_ptr_v3;
1770 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
1771 get_pmu_init_msg_pmu_sw_mg_off_v3;
1772 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
1773 get_pmu_init_msg_pmu_sw_mg_size_v3;
1774 }
1775 g->ops.pmu_ver.get_pmu_perfmon_cmd_start_size =
1776 get_pmu_perfmon_cmd_start_size_v3;
1777 g->ops.pmu_ver.get_perfmon_cmd_start_offsetofvar =
1778 get_perfmon_cmd_start_offsetofvar_v3;
1779 g->ops.pmu_ver.perfmon_start_set_cmd_type =
1780 perfmon_start_set_cmd_type_v3;
1781 g->ops.pmu_ver.perfmon_start_set_group_id =
1782 perfmon_start_set_group_id_v3;
1783 g->ops.pmu_ver.perfmon_start_set_state_id =
1784 perfmon_start_set_state_id_v3;
1785 g->ops.pmu_ver.perfmon_start_set_flags =
1786 perfmon_start_set_flags_v3;
1787 g->ops.pmu_ver.perfmon_start_get_flags =
1788 perfmon_start_get_flags_v3;
1789 g->ops.pmu_ver.get_pmu_perfmon_cmd_init_size =
1790 get_pmu_perfmon_cmd_init_size_v3;
1791 g->ops.pmu_ver.get_perfmon_cmd_init_offsetofvar =
1792 get_perfmon_cmd_init_offsetofvar_v3;
1793 g->ops.pmu_ver.perfmon_cmd_init_set_sample_buffer =
1794 perfmon_cmd_init_set_sample_buffer_v3;
1795 g->ops.pmu_ver.perfmon_cmd_init_set_dec_cnt =
1796 perfmon_cmd_init_set_dec_cnt_v3;
1797 g->ops.pmu_ver.perfmon_cmd_init_set_base_cnt_id =
1798 perfmon_cmd_init_set_base_cnt_id_v3;
1799 g->ops.pmu_ver.perfmon_cmd_init_set_samp_period_us =
1800 perfmon_cmd_init_set_samp_period_us_v3;
1801 g->ops.pmu_ver.perfmon_cmd_init_set_num_cnt =
1802 perfmon_cmd_init_set_num_cnt_v3;
1803 g->ops.pmu_ver.perfmon_cmd_init_set_mov_avg =
1804 perfmon_cmd_init_set_mov_avg_v3;
1805 g->ops.pmu_ver.get_pmu_seq_in_a_ptr =
1806 get_pmu_sequence_in_alloc_ptr_v3;
1807 g->ops.pmu_ver.get_pmu_seq_out_a_ptr =
1808 get_pmu_sequence_out_alloc_ptr_v3;
1809 break;
1810 case APP_VERSION_GM20B_5:
1811 case APP_VERSION_GM20B_4:
1812 g->ops.pmu_ver.pg_cmd_eng_buf_load_size =
1813 pg_cmd_eng_buf_load_size_v0;
1814 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_cmd_type =
1815 pg_cmd_eng_buf_load_set_cmd_type_v0;
1816 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_engine_id =
1817 pg_cmd_eng_buf_load_set_engine_id_v0;
1818 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_idx =
1819 pg_cmd_eng_buf_load_set_buf_idx_v0;
1820 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_pad =
1821 pg_cmd_eng_buf_load_set_pad_v0;
1822 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_size =
1823 pg_cmd_eng_buf_load_set_buf_size_v0;
1824 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_base =
1825 pg_cmd_eng_buf_load_set_dma_base_v0;
1826 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_offset =
1827 pg_cmd_eng_buf_load_set_dma_offset_v0;
1828 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_idx =
1829 pg_cmd_eng_buf_load_set_dma_idx_v0;
1830 g->ops.pmu_ver.get_perfmon_cntr_ptr = get_perfmon_cntr_ptr_v2;
1831 g->ops.pmu_ver.set_perfmon_cntr_ut = set_perfmon_cntr_ut_v2;
1832 g->ops.pmu_ver.set_perfmon_cntr_lt = set_perfmon_cntr_lt_v2;
1833 g->ops.pmu_ver.set_perfmon_cntr_valid =
1834 set_perfmon_cntr_valid_v2;
1835 g->ops.pmu_ver.set_perfmon_cntr_index =
1836 set_perfmon_cntr_index_v2;
1837 g->ops.pmu_ver.set_perfmon_cntr_group_id =
1838 set_perfmon_cntr_group_id_v2;
1839 g->ops.pmu_ver.get_perfmon_cntr_sz = pmu_perfmon_cntr_sz_v2;
1840 g->pmu_ver_cmd_id_zbc_table_update = 16;
1841 __nvgpu_set_enabled(g, NVGPU_PMU_ZBC_SAVE, true);
1842 g->ops.pmu_ver.get_pmu_cmdline_args_size =
1843 pmu_cmdline_size_v3;
1844 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq =
1845 set_pmu_cmdline_args_cpufreq_v3;
1846 g->ops.pmu_ver.set_pmu_cmdline_args_secure_mode =
1847 set_pmu_cmdline_args_secure_mode_v3;
1848 g->ops.pmu_ver.set_pmu_cmdline_args_trace_size =
1849 set_pmu_cmdline_args_falctracesize_v3;
1850 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base =
1851 set_pmu_cmdline_args_falctracedmabase_v3;
1852 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx =
1853 set_pmu_cmdline_args_falctracedmaidx_v3;
1854 g->ops.pmu_ver.get_pmu_cmdline_args_ptr =
1855 get_pmu_cmdline_args_ptr_v3;
1856 g->ops.pmu_ver.get_pmu_allocation_struct_size =
1857 get_pmu_allocation_size_v1;
1858 g->ops.pmu_ver.set_pmu_allocation_ptr =
1859 set_pmu_allocation_ptr_v1;
1860 g->ops.pmu_ver.pmu_allocation_set_dmem_size =
1861 pmu_allocation_set_dmem_size_v1;
1862 g->ops.pmu_ver.pmu_allocation_get_dmem_size =
1863 pmu_allocation_get_dmem_size_v1;
1864 g->ops.pmu_ver.pmu_allocation_get_dmem_offset =
1865 pmu_allocation_get_dmem_offset_v1;
1866 g->ops.pmu_ver.pmu_allocation_get_dmem_offset_addr =
1867 pmu_allocation_get_dmem_offset_addr_v1;
1868 g->ops.pmu_ver.pmu_allocation_set_dmem_offset =
1869 pmu_allocation_set_dmem_offset_v1;
1870 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
1871 get_pmu_init_msg_pmu_queue_params_v1;
1872 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
1873 get_pmu_msg_pmu_init_msg_ptr_v1;
1874 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
1875 get_pmu_init_msg_pmu_sw_mg_off_v1;
1876 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
1877 get_pmu_init_msg_pmu_sw_mg_size_v1;
1878 g->ops.pmu_ver.get_pmu_perfmon_cmd_start_size =
1879 get_pmu_perfmon_cmd_start_size_v1;
1880 g->ops.pmu_ver.get_perfmon_cmd_start_offsetofvar =
1881 get_perfmon_cmd_start_offsetofvar_v1;
1882 g->ops.pmu_ver.perfmon_start_set_cmd_type =
1883 perfmon_start_set_cmd_type_v1;
1884 g->ops.pmu_ver.perfmon_start_set_group_id =
1885 perfmon_start_set_group_id_v1;
1886 g->ops.pmu_ver.perfmon_start_set_state_id =
1887 perfmon_start_set_state_id_v1;
1888 g->ops.pmu_ver.perfmon_start_set_flags =
1889 perfmon_start_set_flags_v1;
1890 g->ops.pmu_ver.perfmon_start_get_flags =
1891 perfmon_start_get_flags_v1;
1892 g->ops.pmu_ver.get_pmu_perfmon_cmd_init_size =
1893 get_pmu_perfmon_cmd_init_size_v1;
1894 g->ops.pmu_ver.get_perfmon_cmd_init_offsetofvar =
1895 get_perfmon_cmd_init_offsetofvar_v1;
1896 g->ops.pmu_ver.perfmon_cmd_init_set_sample_buffer =
1897 perfmon_cmd_init_set_sample_buffer_v1;
1898 g->ops.pmu_ver.perfmon_cmd_init_set_dec_cnt =
1899 perfmon_cmd_init_set_dec_cnt_v1;
1900 g->ops.pmu_ver.perfmon_cmd_init_set_base_cnt_id =
1901 perfmon_cmd_init_set_base_cnt_id_v1;
1902 g->ops.pmu_ver.perfmon_cmd_init_set_samp_period_us =
1903 perfmon_cmd_init_set_samp_period_us_v1;
1904 g->ops.pmu_ver.perfmon_cmd_init_set_num_cnt =
1905 perfmon_cmd_init_set_num_cnt_v1;
1906 g->ops.pmu_ver.perfmon_cmd_init_set_mov_avg =
1907 perfmon_cmd_init_set_mov_avg_v1;
1908 g->ops.pmu_ver.get_pmu_seq_in_a_ptr =
1909 get_pmu_sequence_in_alloc_ptr_v1;
1910 g->ops.pmu_ver.get_pmu_seq_out_a_ptr =
1911 get_pmu_sequence_out_alloc_ptr_v1;
1912 break;
1913 case APP_VERSION_GM20B_3:
1914 case APP_VERSION_GM20B_2:
1915 g->ops.pmu_ver.pg_cmd_eng_buf_load_size =
1916 pg_cmd_eng_buf_load_size_v0;
1917 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_cmd_type =
1918 pg_cmd_eng_buf_load_set_cmd_type_v0;
1919 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_engine_id =
1920 pg_cmd_eng_buf_load_set_engine_id_v0;
1921 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_idx =
1922 pg_cmd_eng_buf_load_set_buf_idx_v0;
1923 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_pad =
1924 pg_cmd_eng_buf_load_set_pad_v0;
1925 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_size =
1926 pg_cmd_eng_buf_load_set_buf_size_v0;
1927 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_base =
1928 pg_cmd_eng_buf_load_set_dma_base_v0;
1929 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_offset =
1930 pg_cmd_eng_buf_load_set_dma_offset_v0;
1931 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_idx =
1932 pg_cmd_eng_buf_load_set_dma_idx_v0;
1933 g->ops.pmu_ver.get_perfmon_cntr_ptr = get_perfmon_cntr_ptr_v2;
1934 g->ops.pmu_ver.set_perfmon_cntr_ut = set_perfmon_cntr_ut_v2;
1935 g->ops.pmu_ver.set_perfmon_cntr_lt = set_perfmon_cntr_lt_v2;
1936 g->ops.pmu_ver.set_perfmon_cntr_valid =
1937 set_perfmon_cntr_valid_v2;
1938 g->ops.pmu_ver.set_perfmon_cntr_index =
1939 set_perfmon_cntr_index_v2;
1940 g->ops.pmu_ver.set_perfmon_cntr_group_id =
1941 set_perfmon_cntr_group_id_v2;
1942 g->ops.pmu_ver.get_perfmon_cntr_sz = pmu_perfmon_cntr_sz_v2;
1943 g->pmu_ver_cmd_id_zbc_table_update = 16;
1944 __nvgpu_set_enabled(g, NVGPU_PMU_ZBC_SAVE, true);
1945 g->ops.pmu_ver.get_pmu_cmdline_args_size =
1946 pmu_cmdline_size_v2;
1947 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq =
1948 set_pmu_cmdline_args_cpufreq_v2;
1949 g->ops.pmu_ver.set_pmu_cmdline_args_secure_mode =
1950 set_pmu_cmdline_args_secure_mode_v2;
1951 g->ops.pmu_ver.set_pmu_cmdline_args_trace_size =
1952 set_pmu_cmdline_args_falctracesize_v2;
1953 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base =
1954 set_pmu_cmdline_args_falctracedmabase_v2;
1955 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx =
1956 set_pmu_cmdline_args_falctracedmaidx_v2;
1957 g->ops.pmu_ver.get_pmu_cmdline_args_ptr =
1958 get_pmu_cmdline_args_ptr_v2;
1959 g->ops.pmu_ver.get_pmu_allocation_struct_size =
1960 get_pmu_allocation_size_v1;
1961 g->ops.pmu_ver.set_pmu_allocation_ptr =
1962 set_pmu_allocation_ptr_v1;
1963 g->ops.pmu_ver.pmu_allocation_set_dmem_size =
1964 pmu_allocation_set_dmem_size_v1;
1965 g->ops.pmu_ver.pmu_allocation_get_dmem_size =
1966 pmu_allocation_get_dmem_size_v1;
1967 g->ops.pmu_ver.pmu_allocation_get_dmem_offset =
1968 pmu_allocation_get_dmem_offset_v1;
1969 g->ops.pmu_ver.pmu_allocation_get_dmem_offset_addr =
1970 pmu_allocation_get_dmem_offset_addr_v1;
1971 g->ops.pmu_ver.pmu_allocation_set_dmem_offset =
1972 pmu_allocation_set_dmem_offset_v1;
1973 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
1974 get_pmu_init_msg_pmu_queue_params_v1;
1975 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
1976 get_pmu_msg_pmu_init_msg_ptr_v1;
1977 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
1978 get_pmu_init_msg_pmu_sw_mg_off_v1;
1979 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
1980 get_pmu_init_msg_pmu_sw_mg_size_v1;
1981 g->ops.pmu_ver.get_pmu_perfmon_cmd_start_size =
1982 get_pmu_perfmon_cmd_start_size_v1;
1983 g->ops.pmu_ver.get_perfmon_cmd_start_offsetofvar =
1984 get_perfmon_cmd_start_offsetofvar_v1;
1985 g->ops.pmu_ver.perfmon_start_set_cmd_type =
1986 perfmon_start_set_cmd_type_v1;
1987 g->ops.pmu_ver.perfmon_start_set_group_id =
1988 perfmon_start_set_group_id_v1;
1989 g->ops.pmu_ver.perfmon_start_set_state_id =
1990 perfmon_start_set_state_id_v1;
1991 g->ops.pmu_ver.perfmon_start_set_flags =
1992 perfmon_start_set_flags_v1;
1993 g->ops.pmu_ver.perfmon_start_get_flags =
1994 perfmon_start_get_flags_v1;
1995 g->ops.pmu_ver.get_pmu_perfmon_cmd_init_size =
1996 get_pmu_perfmon_cmd_init_size_v1;
1997 g->ops.pmu_ver.get_perfmon_cmd_init_offsetofvar =
1998 get_perfmon_cmd_init_offsetofvar_v1;
1999 g->ops.pmu_ver.perfmon_cmd_init_set_sample_buffer =
2000 perfmon_cmd_init_set_sample_buffer_v1;
2001 g->ops.pmu_ver.perfmon_cmd_init_set_dec_cnt =
2002 perfmon_cmd_init_set_dec_cnt_v1;
2003 g->ops.pmu_ver.perfmon_cmd_init_set_base_cnt_id =
2004 perfmon_cmd_init_set_base_cnt_id_v1;
2005 g->ops.pmu_ver.perfmon_cmd_init_set_samp_period_us =
2006 perfmon_cmd_init_set_samp_period_us_v1;
2007 g->ops.pmu_ver.perfmon_cmd_init_set_num_cnt =
2008 perfmon_cmd_init_set_num_cnt_v1;
2009 g->ops.pmu_ver.perfmon_cmd_init_set_mov_avg =
2010 perfmon_cmd_init_set_mov_avg_v1;
2011 g->ops.pmu_ver.get_pmu_seq_in_a_ptr =
2012 get_pmu_sequence_in_alloc_ptr_v1;
2013 g->ops.pmu_ver.get_pmu_seq_out_a_ptr =
2014 get_pmu_sequence_out_alloc_ptr_v1;
2015 break;
2016 case APP_VERSION_GM20B_1:
2017 case APP_VERSION_GM20B:
2018 case APP_VERSION_1:
2019 case APP_VERSION_2:
2020 case APP_VERSION_3:
2021 g->ops.pmu_ver.pg_cmd_eng_buf_load_size =
2022 pg_cmd_eng_buf_load_size_v0;
2023 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_cmd_type =
2024 pg_cmd_eng_buf_load_set_cmd_type_v0;
2025 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_engine_id =
2026 pg_cmd_eng_buf_load_set_engine_id_v0;
2027 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_idx =
2028 pg_cmd_eng_buf_load_set_buf_idx_v0;
2029 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_pad =
2030 pg_cmd_eng_buf_load_set_pad_v0;
2031 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_size =
2032 pg_cmd_eng_buf_load_set_buf_size_v0;
2033 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_base =
2034 pg_cmd_eng_buf_load_set_dma_base_v0;
2035 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_offset =
2036 pg_cmd_eng_buf_load_set_dma_offset_v0;
2037 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_idx =
2038 pg_cmd_eng_buf_load_set_dma_idx_v0;
2039 g->pmu_ver_cmd_id_zbc_table_update = 16;
2040 __nvgpu_set_enabled(g, NVGPU_PMU_ZBC_SAVE, true);
2041 g->ops.pmu_ver.get_perfmon_cntr_ptr = get_perfmon_cntr_ptr_v0;
2042 g->ops.pmu_ver.set_perfmon_cntr_ut = set_perfmon_cntr_ut_v0;
2043 g->ops.pmu_ver.set_perfmon_cntr_lt = set_perfmon_cntr_lt_v0;
2044 g->ops.pmu_ver.set_perfmon_cntr_valid =
2045 set_perfmon_cntr_valid_v0;
2046 g->ops.pmu_ver.set_perfmon_cntr_index =
2047 set_perfmon_cntr_index_v0;
2048 g->ops.pmu_ver.set_perfmon_cntr_group_id =
2049 set_perfmon_cntr_group_id_v0;
2050 g->ops.pmu_ver.get_perfmon_cntr_sz = pmu_perfmon_cntr_sz_v0;
2051 g->ops.pmu_ver.get_pmu_cmdline_args_size =
2052 pmu_cmdline_size_v1;
2053 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq =
2054 set_pmu_cmdline_args_cpufreq_v1;
2055 g->ops.pmu_ver.set_pmu_cmdline_args_secure_mode =
2056 set_pmu_cmdline_args_secure_mode_v1;
2057 g->ops.pmu_ver.set_pmu_cmdline_args_trace_size =
2058 set_pmu_cmdline_args_falctracesize_v1;
2059 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base =
2060 set_pmu_cmdline_args_falctracedmabase_v1;
2061 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx =
2062 set_pmu_cmdline_args_falctracedmaidx_v1;
2063 g->ops.pmu_ver.get_pmu_cmdline_args_ptr =
2064 get_pmu_cmdline_args_ptr_v1;
2065 g->ops.pmu_ver.get_pmu_allocation_struct_size =
2066 get_pmu_allocation_size_v1;
2067 g->ops.pmu_ver.set_pmu_allocation_ptr =
2068 set_pmu_allocation_ptr_v1;
2069 g->ops.pmu_ver.pmu_allocation_set_dmem_size =
2070 pmu_allocation_set_dmem_size_v1;
2071 g->ops.pmu_ver.pmu_allocation_get_dmem_size =
2072 pmu_allocation_get_dmem_size_v1;
2073 g->ops.pmu_ver.pmu_allocation_get_dmem_offset =
2074 pmu_allocation_get_dmem_offset_v1;
2075 g->ops.pmu_ver.pmu_allocation_get_dmem_offset_addr =
2076 pmu_allocation_get_dmem_offset_addr_v1;
2077 g->ops.pmu_ver.pmu_allocation_set_dmem_offset =
2078 pmu_allocation_set_dmem_offset_v1;
2079 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
2080 get_pmu_init_msg_pmu_queue_params_v1;
2081 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
2082 get_pmu_msg_pmu_init_msg_ptr_v1;
2083 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
2084 get_pmu_init_msg_pmu_sw_mg_off_v1;
2085 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
2086 get_pmu_init_msg_pmu_sw_mg_size_v1;
2087 g->ops.pmu_ver.get_pmu_perfmon_cmd_start_size =
2088 get_pmu_perfmon_cmd_start_size_v1;
2089 g->ops.pmu_ver.get_perfmon_cmd_start_offsetofvar =
2090 get_perfmon_cmd_start_offsetofvar_v1;
2091 g->ops.pmu_ver.perfmon_start_set_cmd_type =
2092 perfmon_start_set_cmd_type_v1;
2093 g->ops.pmu_ver.perfmon_start_set_group_id =
2094 perfmon_start_set_group_id_v1;
2095 g->ops.pmu_ver.perfmon_start_set_state_id =
2096 perfmon_start_set_state_id_v1;
2097 g->ops.pmu_ver.perfmon_start_set_flags =
2098 perfmon_start_set_flags_v1;
2099 g->ops.pmu_ver.perfmon_start_get_flags =
2100 perfmon_start_get_flags_v1;
2101 g->ops.pmu_ver.get_pmu_perfmon_cmd_init_size =
2102 get_pmu_perfmon_cmd_init_size_v1;
2103 g->ops.pmu_ver.get_perfmon_cmd_init_offsetofvar =
2104 get_perfmon_cmd_init_offsetofvar_v1;
2105 g->ops.pmu_ver.perfmon_cmd_init_set_sample_buffer =
2106 perfmon_cmd_init_set_sample_buffer_v1;
2107 g->ops.pmu_ver.perfmon_cmd_init_set_dec_cnt =
2108 perfmon_cmd_init_set_dec_cnt_v1;
2109 g->ops.pmu_ver.perfmon_cmd_init_set_base_cnt_id =
2110 perfmon_cmd_init_set_base_cnt_id_v1;
2111 g->ops.pmu_ver.perfmon_cmd_init_set_samp_period_us =
2112 perfmon_cmd_init_set_samp_period_us_v1;
2113 g->ops.pmu_ver.perfmon_cmd_init_set_num_cnt =
2114 perfmon_cmd_init_set_num_cnt_v1;
2115 g->ops.pmu_ver.perfmon_cmd_init_set_mov_avg =
2116 perfmon_cmd_init_set_mov_avg_v1;
2117 g->ops.pmu_ver.get_pmu_seq_in_a_ptr =
2118 get_pmu_sequence_in_alloc_ptr_v1;
2119 g->ops.pmu_ver.get_pmu_seq_out_a_ptr =
2120 get_pmu_sequence_out_alloc_ptr_v1;
2121 break;
2122 case APP_VERSION_0:
2123 g->ops.pmu_ver.pg_cmd_eng_buf_load_size =
2124 pg_cmd_eng_buf_load_size_v0;
2125 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_cmd_type =
2126 pg_cmd_eng_buf_load_set_cmd_type_v0;
2127 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_engine_id =
2128 pg_cmd_eng_buf_load_set_engine_id_v0;
2129 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_idx =
2130 pg_cmd_eng_buf_load_set_buf_idx_v0;
2131 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_pad =
2132 pg_cmd_eng_buf_load_set_pad_v0;
2133 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_size =
2134 pg_cmd_eng_buf_load_set_buf_size_v0;
2135 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_base =
2136 pg_cmd_eng_buf_load_set_dma_base_v0;
2137 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_offset =
2138 pg_cmd_eng_buf_load_set_dma_offset_v0;
2139 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_idx =
2140 pg_cmd_eng_buf_load_set_dma_idx_v0;
2141 g->pmu_ver_cmd_id_zbc_table_update = 14;
2142 __nvgpu_set_enabled(g, NVGPU_PMU_ZBC_SAVE, true);
2143 g->ops.pmu_ver.get_perfmon_cntr_ptr = get_perfmon_cntr_ptr_v0;
2144 g->ops.pmu_ver.set_perfmon_cntr_ut = set_perfmon_cntr_ut_v0;
2145 g->ops.pmu_ver.set_perfmon_cntr_lt = set_perfmon_cntr_lt_v0;
2146 g->ops.pmu_ver.set_perfmon_cntr_valid =
2147 set_perfmon_cntr_valid_v0;
2148 g->ops.pmu_ver.set_perfmon_cntr_index =
2149 set_perfmon_cntr_index_v0;
2150 g->ops.pmu_ver.set_perfmon_cntr_group_id =
2151 set_perfmon_cntr_group_id_v0;
2152 g->ops.pmu_ver.get_perfmon_cntr_sz = pmu_perfmon_cntr_sz_v0;
2153 g->ops.pmu_ver.get_pmu_cmdline_args_size =
2154 pmu_cmdline_size_v0;
2155 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq =
2156 set_pmu_cmdline_args_cpufreq_v0;
2157 g->ops.pmu_ver.set_pmu_cmdline_args_secure_mode =
2158 NULL;
2159 g->ops.pmu_ver.get_pmu_cmdline_args_ptr =
2160 get_pmu_cmdline_args_ptr_v0;
2161 g->ops.pmu_ver.get_pmu_allocation_struct_size =
2162 get_pmu_allocation_size_v0;
2163 g->ops.pmu_ver.set_pmu_allocation_ptr =
2164 set_pmu_allocation_ptr_v0;
2165 g->ops.pmu_ver.pmu_allocation_set_dmem_size =
2166 pmu_allocation_set_dmem_size_v0;
2167 g->ops.pmu_ver.pmu_allocation_get_dmem_size =
2168 pmu_allocation_get_dmem_size_v0;
2169 g->ops.pmu_ver.pmu_allocation_get_dmem_offset =
2170 pmu_allocation_get_dmem_offset_v0;
2171 g->ops.pmu_ver.pmu_allocation_get_dmem_offset_addr =
2172 pmu_allocation_get_dmem_offset_addr_v0;
2173 g->ops.pmu_ver.pmu_allocation_set_dmem_offset =
2174 pmu_allocation_set_dmem_offset_v0;
2175 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
2176 get_pmu_init_msg_pmu_queue_params_v0;
2177 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
2178 get_pmu_msg_pmu_init_msg_ptr_v0;
2179 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
2180 get_pmu_init_msg_pmu_sw_mg_off_v0;
2181 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
2182 get_pmu_init_msg_pmu_sw_mg_size_v0;
2183 g->ops.pmu_ver.get_pmu_perfmon_cmd_start_size =
2184 get_pmu_perfmon_cmd_start_size_v0;
2185 g->ops.pmu_ver.get_perfmon_cmd_start_offsetofvar =
2186 get_perfmon_cmd_start_offsetofvar_v0;
2187 g->ops.pmu_ver.perfmon_start_set_cmd_type =
2188 perfmon_start_set_cmd_type_v0;
2189 g->ops.pmu_ver.perfmon_start_set_group_id =
2190 perfmon_start_set_group_id_v0;
2191 g->ops.pmu_ver.perfmon_start_set_state_id =
2192 perfmon_start_set_state_id_v0;
2193 g->ops.pmu_ver.perfmon_start_set_flags =
2194 perfmon_start_set_flags_v0;
2195 g->ops.pmu_ver.perfmon_start_get_flags =
2196 perfmon_start_get_flags_v0;
2197 g->ops.pmu_ver.get_pmu_perfmon_cmd_init_size =
2198 get_pmu_perfmon_cmd_init_size_v0;
2199 g->ops.pmu_ver.get_perfmon_cmd_init_offsetofvar =
2200 get_perfmon_cmd_init_offsetofvar_v0;
2201 g->ops.pmu_ver.perfmon_cmd_init_set_sample_buffer =
2202 perfmon_cmd_init_set_sample_buffer_v0;
2203 g->ops.pmu_ver.perfmon_cmd_init_set_dec_cnt =
2204 perfmon_cmd_init_set_dec_cnt_v0;
2205 g->ops.pmu_ver.perfmon_cmd_init_set_base_cnt_id =
2206 perfmon_cmd_init_set_base_cnt_id_v0;
2207 g->ops.pmu_ver.perfmon_cmd_init_set_samp_period_us =
2208 perfmon_cmd_init_set_samp_period_us_v0;
2209 g->ops.pmu_ver.perfmon_cmd_init_set_num_cnt =
2210 perfmon_cmd_init_set_num_cnt_v0;
2211 g->ops.pmu_ver.perfmon_cmd_init_set_mov_avg =
2212 perfmon_cmd_init_set_mov_avg_v0;
2213 g->ops.pmu_ver.get_pmu_seq_in_a_ptr =
2214 get_pmu_sequence_in_alloc_ptr_v0;
2215 g->ops.pmu_ver.get_pmu_seq_out_a_ptr =
2216 get_pmu_sequence_out_alloc_ptr_v0;
2217 break;
2218 default:
2219 nvgpu_err(g, "PMU code version not supported version: %d\n",
2220 pmu->desc->app_version);
2221 err = -EINVAL;
2222 }
2223 pv->set_perfmon_cntr_index(pmu, 3); /* GR & CE2 */
2224 pv->set_perfmon_cntr_group_id(pmu, PMU_DOMAIN_GROUP_PSTATE);
2225
2226 return err;
2227}
2228
2229static void nvgpu_remove_pmu_support(struct nvgpu_pmu *pmu)
2230{
2231 struct gk20a *g = gk20a_from_pmu(pmu);
2232 struct mm_gk20a *mm = &g->mm;
2233 struct vm_gk20a *vm = mm->pmu.vm;
2234 struct boardobj *pboardobj, *pboardobj_tmp;
2235 struct boardobjgrp *pboardobjgrp, *pboardobjgrp_tmp;
2236
2237 nvgpu_log_fn(g, " ");
2238
2239 if (nvgpu_alloc_initialized(&pmu->dmem))
2240 nvgpu_alloc_destroy(&pmu->dmem);
2241
2242 nvgpu_list_for_each_entry_safe(pboardobjgrp, pboardobjgrp_tmp,
2243 &g->boardobjgrp_head, boardobjgrp, node) {
2244 pboardobjgrp->destruct(pboardobjgrp);
2245 }
2246
2247 nvgpu_list_for_each_entry_safe(pboardobj, pboardobj_tmp,
2248 &g->boardobj_head, boardobj, node) {
2249 pboardobj->destruct(pboardobj);
2250 }
2251
2252 if (pmu->fw)
2253 nvgpu_release_firmware(g, pmu->fw);
2254
2255 if (g->acr.pmu_fw)
2256 nvgpu_release_firmware(g, g->acr.pmu_fw);
2257
2258 if (g->acr.pmu_desc)
2259 nvgpu_release_firmware(g, g->acr.pmu_desc);
2260
2261 if (g->acr.acr_fw)
2262 nvgpu_release_firmware(g, g->acr.acr_fw);
2263
2264 if (g->acr.hsbl_fw)
2265 nvgpu_release_firmware(g, g->acr.hsbl_fw);
2266
2267 nvgpu_dma_unmap_free(vm, &g->acr.acr_ucode);
2268 nvgpu_dma_unmap_free(vm, &g->acr.hsbl_ucode);
2269
2270 nvgpu_dma_unmap_free(vm, &pmu->seq_buf);
2271
2272 nvgpu_mutex_destroy(&pmu->elpg_mutex);
2273 nvgpu_mutex_destroy(&pmu->pg_mutex);
2274 nvgpu_mutex_destroy(&pmu->isr_mutex);
2275 nvgpu_mutex_destroy(&pmu->pmu_copy_lock);
2276 nvgpu_mutex_destroy(&pmu->pmu_seq_lock);
2277}
2278
2279int nvgpu_init_pmu_fw_support(struct nvgpu_pmu *pmu)
2280{
2281 struct gk20a *g = gk20a_from_pmu(pmu);
2282 int err = 0;
2283
2284 nvgpu_log_fn(g, " ");
2285
2286 err = nvgpu_mutex_init(&pmu->elpg_mutex);
2287 if (err)
2288 return err;
2289
2290 err = nvgpu_mutex_init(&pmu->pg_mutex);
2291 if (err)
2292 goto fail_elpg;
2293
2294 err = nvgpu_mutex_init(&pmu->isr_mutex);
2295 if (err)
2296 goto fail_pg;
2297
2298 err = nvgpu_mutex_init(&pmu->pmu_copy_lock);
2299 if (err)
2300 goto fail_isr;
2301
2302 err = nvgpu_mutex_init(&pmu->pmu_seq_lock);
2303 if (err)
2304 goto fail_pmu_copy;
2305
2306 pmu->remove_support = nvgpu_remove_pmu_support;
2307
2308 err = nvgpu_init_pmu_fw_ver_ops(pmu);
2309 if (err)
2310 goto fail_pmu_seq;
2311
2312 goto exit;
2313
2314fail_pmu_seq:
2315 nvgpu_mutex_destroy(&pmu->pmu_seq_lock);
2316fail_pmu_copy:
2317 nvgpu_mutex_destroy(&pmu->pmu_copy_lock);
2318fail_isr:
2319 nvgpu_mutex_destroy(&pmu->isr_mutex);
2320fail_pg:
2321 nvgpu_mutex_destroy(&pmu->pg_mutex);
2322fail_elpg:
2323 nvgpu_mutex_destroy(&pmu->elpg_mutex);
2324exit:
2325 return err;
2326}
2327
2328int nvgpu_pmu_prepare_ns_ucode_blob(struct gk20a *g)
2329{
2330 struct nvgpu_pmu *pmu = &g->pmu;
2331 int err = 0;
2332 struct mm_gk20a *mm = &g->mm;
2333 struct vm_gk20a *vm = mm->pmu.vm;
2334
2335 nvgpu_log_fn(g, " ");
2336
2337 if (pmu->fw)
2338 return nvgpu_init_pmu_fw_support(pmu);
2339
2340 pmu->fw = nvgpu_request_firmware(g, NVGPU_PMU_NS_UCODE_IMAGE, 0);
2341 if (!pmu->fw) {
2342 nvgpu_err(g, "failed to load pmu ucode!!");
2343 return err;
2344 }
2345
2346 nvgpu_log_fn(g, "firmware loaded");
2347
2348 pmu->desc = (struct pmu_ucode_desc *)pmu->fw->data;
2349 pmu->ucode_image = (u32 *)((u8 *)pmu->desc +
2350 pmu->desc->descriptor_size);
2351
2352 err = nvgpu_dma_alloc_map_sys(vm, GK20A_PMU_UCODE_SIZE_MAX,
2353 &pmu->ucode);
2354 if (err)
2355 goto err_release_fw;
2356
2357 nvgpu_mem_wr_n(g, &pmu->ucode, 0, pmu->ucode_image,
2358 pmu->desc->app_start_offset + pmu->desc->app_size);
2359
2360 return nvgpu_init_pmu_fw_support(pmu);
2361
2362 err_release_fw:
2363 nvgpu_release_firmware(g, pmu->fw);
2364 pmu->fw = NULL;
2365
2366 return err;
2367}
2368
diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_ipc.c b/drivers/gpu/nvgpu/common/pmu/pmu_ipc.c
new file mode 100644
index 00000000..4c706e57
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/pmu/pmu_ipc.c
@@ -0,0 +1,907 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/enabled.h>
24#include <nvgpu/pmu.h>
25#include <nvgpu/log.h>
26#include <nvgpu/timers.h>
27#include <nvgpu/bug.h>
28#include <nvgpu/pmuif/nvgpu_gpmu_cmdif.h>
29
30#include "gk20a/gk20a.h"
31
32void nvgpu_pmu_seq_init(struct nvgpu_pmu *pmu)
33{
34 u32 i;
35
36 memset(pmu->seq, 0,
37 sizeof(struct pmu_sequence) * PMU_MAX_NUM_SEQUENCES);
38 memset(pmu->pmu_seq_tbl, 0,
39 sizeof(pmu->pmu_seq_tbl));
40
41 for (i = 0; i < PMU_MAX_NUM_SEQUENCES; i++)
42 pmu->seq[i].id = i;
43}
44
45static int pmu_seq_acquire(struct nvgpu_pmu *pmu,
46 struct pmu_sequence **pseq)
47{
48 struct gk20a *g = gk20a_from_pmu(pmu);
49 struct pmu_sequence *seq;
50 u32 index;
51
52 nvgpu_mutex_acquire(&pmu->pmu_seq_lock);
53 index = find_first_zero_bit(pmu->pmu_seq_tbl,
54 sizeof(pmu->pmu_seq_tbl));
55 if (index >= sizeof(pmu->pmu_seq_tbl)) {
56 nvgpu_err(g, "no free sequence available");
57 nvgpu_mutex_release(&pmu->pmu_seq_lock);
58 return -EAGAIN;
59 }
60 set_bit(index, pmu->pmu_seq_tbl);
61 nvgpu_mutex_release(&pmu->pmu_seq_lock);
62
63 seq = &pmu->seq[index];
64 seq->state = PMU_SEQ_STATE_PENDING;
65
66 *pseq = seq;
67 return 0;
68}
69
70static void pmu_seq_release(struct nvgpu_pmu *pmu,
71 struct pmu_sequence *seq)
72{
73 struct gk20a *g = gk20a_from_pmu(pmu);
74
75 seq->state = PMU_SEQ_STATE_FREE;
76 seq->desc = PMU_INVALID_SEQ_DESC;
77 seq->callback = NULL;
78 seq->cb_params = NULL;
79 seq->msg = NULL;
80 seq->out_payload = NULL;
81 g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu,
82 g->ops.pmu_ver.get_pmu_seq_in_a_ptr(seq), 0);
83 g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu,
84 g->ops.pmu_ver.get_pmu_seq_out_a_ptr(seq), 0);
85
86 clear_bit(seq->id, pmu->pmu_seq_tbl);
87}
88/* mutex */
89int nvgpu_pmu_mutex_acquire(struct nvgpu_pmu *pmu, u32 id, u32 *token)
90{
91 struct gk20a *g = gk20a_from_pmu(pmu);
92
93 return g->ops.pmu.pmu_mutex_acquire(pmu, id, token);
94}
95
96int nvgpu_pmu_mutex_release(struct nvgpu_pmu *pmu, u32 id, u32 *token)
97{
98 struct gk20a *g = gk20a_from_pmu(pmu);
99
100 return g->ops.pmu.pmu_mutex_release(pmu, id, token);
101}
102
103/* queue */
104int nvgpu_pmu_queue_init(struct nvgpu_pmu *pmu,
105 u32 id, union pmu_init_msg_pmu *init)
106{
107 struct gk20a *g = gk20a_from_pmu(pmu);
108 struct pmu_queue *queue = &pmu->queue[id];
109 int err;
110
111 err = nvgpu_mutex_init(&queue->mutex);
112 if (err)
113 return err;
114
115 queue->id = id;
116 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params(queue, id, init);
117 queue->mutex_id = id;
118
119 nvgpu_pmu_dbg(g, "queue %d: index %d, offset 0x%08x, size 0x%08x",
120 id, queue->index, queue->offset, queue->size);
121
122 return 0;
123}
124
125static int pmu_queue_head(struct nvgpu_pmu *pmu, struct pmu_queue *queue,
126 u32 *head, bool set)
127{
128 struct gk20a *g = gk20a_from_pmu(pmu);
129
130 return g->ops.pmu.pmu_queue_head(pmu, queue, head, set);
131}
132
133static int pmu_queue_tail(struct nvgpu_pmu *pmu, struct pmu_queue *queue,
134 u32 *tail, bool set)
135{
136 struct gk20a *g = gk20a_from_pmu(pmu);
137
138 return g->ops.pmu.pmu_queue_tail(pmu, queue, tail, set);
139}
140
141static inline void pmu_queue_read(struct nvgpu_pmu *pmu,
142 u32 offset, u8 *dst, u32 size)
143{
144 nvgpu_flcn_copy_from_dmem(pmu->flcn, offset, dst, size, 0);
145}
146
147static inline void pmu_queue_write(struct nvgpu_pmu *pmu,
148 u32 offset, u8 *src, u32 size)
149{
150 nvgpu_flcn_copy_to_dmem(pmu->flcn, offset, src, size, 0);
151}
152
153
154static int pmu_queue_lock(struct nvgpu_pmu *pmu,
155 struct pmu_queue *queue)
156{
157 int err;
158
159 if (PMU_IS_MESSAGE_QUEUE(queue->id))
160 return 0;
161
162 if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
163 nvgpu_mutex_acquire(&queue->mutex);
164 return 0;
165 }
166
167 err = nvgpu_pmu_mutex_acquire(pmu, queue->mutex_id, &queue->mutex_lock);
168 return err;
169}
170
171static int pmu_queue_unlock(struct nvgpu_pmu *pmu,
172 struct pmu_queue *queue)
173{
174 int err;
175
176 if (PMU_IS_MESSAGE_QUEUE(queue->id))
177 return 0;
178
179 if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
180 nvgpu_mutex_release(&queue->mutex);
181 return 0;
182 }
183
184 err = nvgpu_pmu_mutex_release(pmu, queue->mutex_id, &queue->mutex_lock);
185 return err;
186}
187
188/* called by pmu_read_message, no lock */
189bool nvgpu_pmu_queue_is_empty(struct nvgpu_pmu *pmu,
190 struct pmu_queue *queue)
191{
192 u32 head, tail;
193
194 pmu_queue_head(pmu, queue, &head, QUEUE_GET);
195 if (queue->opened && queue->oflag == OFLAG_READ)
196 tail = queue->position;
197 else
198 pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
199
200 return head == tail;
201}
202
203static bool pmu_queue_has_room(struct nvgpu_pmu *pmu,
204 struct pmu_queue *queue, u32 size, bool *need_rewind)
205{
206 u32 head, tail;
207 bool rewind = false;
208 unsigned int free;
209
210 size = ALIGN(size, QUEUE_ALIGNMENT);
211
212 pmu_queue_head(pmu, queue, &head, QUEUE_GET);
213 pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
214 if (head >= tail) {
215 free = queue->offset + queue->size - head;
216 free -= PMU_CMD_HDR_SIZE;
217
218 if (size > free) {
219 rewind = true;
220 head = queue->offset;
221 }
222 }
223
224 if (head < tail)
225 free = tail - head - 1;
226
227 if (need_rewind)
228 *need_rewind = rewind;
229
230 return size <= free;
231}
232
233static int pmu_queue_push(struct nvgpu_pmu *pmu,
234 struct pmu_queue *queue, void *data, u32 size)
235{
236 struct gk20a *g = pmu->g;
237
238 nvgpu_log_fn(g, " ");
239
240 if (!queue->opened && queue->oflag == OFLAG_WRITE) {
241 nvgpu_err(gk20a_from_pmu(pmu), "queue not opened for write");
242 return -EINVAL;
243 }
244
245 pmu_queue_write(pmu, queue->position, data, size);
246 queue->position += ALIGN(size, QUEUE_ALIGNMENT);
247 return 0;
248}
249
250static int pmu_queue_pop(struct nvgpu_pmu *pmu,
251 struct pmu_queue *queue, void *data, u32 size,
252 u32 *bytes_read)
253{
254 u32 head, tail, used;
255
256 *bytes_read = 0;
257
258 if (!queue->opened && queue->oflag == OFLAG_READ) {
259 nvgpu_err(gk20a_from_pmu(pmu), "queue not opened for read");
260 return -EINVAL;
261 }
262
263 pmu_queue_head(pmu, queue, &head, QUEUE_GET);
264 tail = queue->position;
265
266 if (head == tail)
267 return 0;
268
269 if (head > tail)
270 used = head - tail;
271 else
272 used = queue->offset + queue->size - tail;
273
274 if (size > used) {
275 nvgpu_warn(gk20a_from_pmu(pmu),
276 "queue size smaller than request read");
277 size = used;
278 }
279
280 pmu_queue_read(pmu, tail, data, size);
281 queue->position += ALIGN(size, QUEUE_ALIGNMENT);
282 *bytes_read = size;
283 return 0;
284}
285
286static void pmu_queue_rewind(struct nvgpu_pmu *pmu,
287 struct pmu_queue *queue)
288{
289 struct gk20a *g = gk20a_from_pmu(pmu);
290 struct pmu_cmd cmd;
291
292 nvgpu_log_fn(g, " ");
293
294 if (!queue->opened) {
295 nvgpu_err(gk20a_from_pmu(pmu), "queue not opened");
296 return;
297 }
298
299 if (queue->oflag == OFLAG_WRITE) {
300 cmd.hdr.unit_id = PMU_UNIT_REWIND;
301 cmd.hdr.size = PMU_CMD_HDR_SIZE;
302 pmu_queue_push(pmu, queue, &cmd, cmd.hdr.size);
303 nvgpu_pmu_dbg(g, "queue %d rewinded", queue->id);
304 }
305
306 queue->position = queue->offset;
307}
308
309/* open for read and lock the queue */
310static int pmu_queue_open_read(struct nvgpu_pmu *pmu,
311 struct pmu_queue *queue)
312{
313 int err;
314
315 err = pmu_queue_lock(pmu, queue);
316 if (err)
317 return err;
318
319 if (queue->opened)
320 BUG();
321
322 pmu_queue_tail(pmu, queue, &queue->position, QUEUE_GET);
323 queue->oflag = OFLAG_READ;
324 queue->opened = true;
325
326 return 0;
327}
328
329/* open for write and lock the queue
330 * make sure there's enough free space for the write
331 * */
332static int pmu_queue_open_write(struct nvgpu_pmu *pmu,
333 struct pmu_queue *queue, u32 size)
334{
335 struct gk20a *g = gk20a_from_pmu(pmu);
336 bool rewind = false;
337 int err;
338
339 err = pmu_queue_lock(pmu, queue);
340 if (err)
341 return err;
342
343 if (queue->opened)
344 BUG();
345
346 if (!pmu_queue_has_room(pmu, queue, size, &rewind)) {
347 nvgpu_pmu_dbg(g, "queue full: queue-id %d: index %d",
348 queue->id, queue->index);
349 pmu_queue_unlock(pmu, queue);
350 return -EAGAIN;
351 }
352
353 pmu_queue_head(pmu, queue, &queue->position, QUEUE_GET);
354 queue->oflag = OFLAG_WRITE;
355 queue->opened = true;
356
357 if (rewind)
358 pmu_queue_rewind(pmu, queue);
359
360 return 0;
361}
362
363/* close and unlock the queue */
364static int pmu_queue_close(struct nvgpu_pmu *pmu,
365 struct pmu_queue *queue, bool commit)
366{
367 if (!queue->opened)
368 return 0;
369
370 if (commit) {
371 if (queue->oflag == OFLAG_READ)
372 pmu_queue_tail(pmu, queue,
373 &queue->position, QUEUE_SET);
374 else
375 pmu_queue_head(pmu, queue,
376 &queue->position, QUEUE_SET);
377 }
378
379 queue->opened = false;
380
381 pmu_queue_unlock(pmu, queue);
382
383 return 0;
384}
385
386static bool pmu_validate_cmd(struct nvgpu_pmu *pmu, struct pmu_cmd *cmd,
387 struct pmu_msg *msg, struct pmu_payload *payload,
388 u32 queue_id)
389{
390 struct gk20a *g = gk20a_from_pmu(pmu);
391 struct pmu_queue *queue;
392 u32 in_size, out_size;
393
394 if (!PMU_IS_SW_COMMAND_QUEUE(queue_id))
395 goto invalid_cmd;
396
397 queue = &pmu->queue[queue_id];
398 if (cmd->hdr.size < PMU_CMD_HDR_SIZE)
399 goto invalid_cmd;
400
401 if (cmd->hdr.size > (queue->size >> 1))
402 goto invalid_cmd;
403
404 if (msg != NULL && msg->hdr.size < PMU_MSG_HDR_SIZE)
405 goto invalid_cmd;
406
407 if (!PMU_UNIT_ID_IS_VALID(cmd->hdr.unit_id))
408 goto invalid_cmd;
409
410 if (payload == NULL)
411 return true;
412
413 if (payload->in.buf == NULL && payload->out.buf == NULL)
414 goto invalid_cmd;
415
416 if ((payload->in.buf != NULL && payload->in.size == 0) ||
417 (payload->out.buf != NULL && payload->out.size == 0))
418 goto invalid_cmd;
419
420 in_size = PMU_CMD_HDR_SIZE;
421 if (payload->in.buf) {
422 in_size += payload->in.offset;
423 in_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu);
424 }
425
426 out_size = PMU_CMD_HDR_SIZE;
427 if (payload->out.buf) {
428 out_size += payload->out.offset;
429 out_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu);
430 }
431
432 if (in_size > cmd->hdr.size || out_size > cmd->hdr.size)
433 goto invalid_cmd;
434
435
436 if ((payload->in.offset != 0 && payload->in.buf == NULL) ||
437 (payload->out.offset != 0 && payload->out.buf == NULL))
438 goto invalid_cmd;
439
440 return true;
441
442invalid_cmd:
443 nvgpu_err(g, "invalid pmu cmd :\n"
444 "queue_id=%d,\n"
445 "cmd_size=%d, cmd_unit_id=%d, msg=%p, msg_size=%d,\n"
446 "payload in=%p, in_size=%d, in_offset=%d,\n"
447 "payload out=%p, out_size=%d, out_offset=%d",
448 queue_id, cmd->hdr.size, cmd->hdr.unit_id,
449 msg, msg ? msg->hdr.unit_id : ~0,
450 &payload->in, payload->in.size, payload->in.offset,
451 &payload->out, payload->out.size, payload->out.offset);
452
453 return false;
454}
455
456static int pmu_write_cmd(struct nvgpu_pmu *pmu, struct pmu_cmd *cmd,
457 u32 queue_id, unsigned long timeout_ms)
458{
459 struct gk20a *g = gk20a_from_pmu(pmu);
460 struct pmu_queue *queue;
461 struct nvgpu_timeout timeout;
462 int err;
463
464 nvgpu_log_fn(g, " ");
465
466 queue = &pmu->queue[queue_id];
467 nvgpu_timeout_init(g, &timeout, timeout_ms, NVGPU_TIMER_CPU_TIMER);
468
469 do {
470 err = pmu_queue_open_write(pmu, queue, cmd->hdr.size);
471 if (err == -EAGAIN && !nvgpu_timeout_expired(&timeout))
472 nvgpu_usleep_range(1000, 2000);
473 else
474 break;
475 } while (1);
476
477 if (err)
478 goto clean_up;
479
480 pmu_queue_push(pmu, queue, cmd, cmd->hdr.size);
481
482
483 err = pmu_queue_close(pmu, queue, true);
484
485clean_up:
486 if (err)
487 nvgpu_err(g, "fail to write cmd to queue %d", queue_id);
488 else
489 nvgpu_log_fn(g, "done");
490
491 return err;
492}
493
494int nvgpu_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd,
495 struct pmu_msg *msg, struct pmu_payload *payload,
496 u32 queue_id, pmu_callback callback, void *cb_param,
497 u32 *seq_desc, unsigned long timeout)
498{
499 struct nvgpu_pmu *pmu = &g->pmu;
500 struct pmu_v *pv = &g->ops.pmu_ver;
501 struct pmu_sequence *seq;
502 void *in = NULL, *out = NULL;
503 int err;
504
505 nvgpu_log_fn(g, " ");
506
507 if ((!cmd) || (!seq_desc) || (!pmu->pmu_ready)) {
508 if (!cmd)
509 nvgpu_warn(g, "%s(): PMU cmd buffer is NULL", __func__);
510 else if (!seq_desc)
511 nvgpu_warn(g, "%s(): Seq descriptor is NULL", __func__);
512 else
513 nvgpu_warn(g, "%s(): PMU is not ready", __func__);
514
515 WARN_ON(1);
516 return -EINVAL;
517 }
518
519 if (!pmu_validate_cmd(pmu, cmd, msg, payload, queue_id))
520 return -EINVAL;
521
522 err = pmu_seq_acquire(pmu, &seq);
523 if (err)
524 return err;
525
526 cmd->hdr.seq_id = seq->id;
527
528 cmd->hdr.ctrl_flags = 0;
529 cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_STATUS;
530 cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_INTR;
531
532 seq->callback = callback;
533 seq->cb_params = cb_param;
534 seq->msg = msg;
535 seq->out_payload = NULL;
536 seq->desc = pmu->next_seq_desc++;
537
538 if (payload)
539 seq->out_payload = payload->out.buf;
540
541 *seq_desc = seq->desc;
542
543 if (payload && payload->in.offset != 0) {
544 pv->set_pmu_allocation_ptr(pmu, &in,
545 ((u8 *)&cmd->cmd + payload->in.offset));
546
547 if (payload->in.buf != payload->out.buf)
548 pv->pmu_allocation_set_dmem_size(pmu, in,
549 (u16)payload->in.size);
550 else
551 pv->pmu_allocation_set_dmem_size(pmu, in,
552 (u16)max(payload->in.size, payload->out.size));
553
554 *(pv->pmu_allocation_get_dmem_offset_addr(pmu, in)) =
555 nvgpu_alloc(&pmu->dmem,
556 pv->pmu_allocation_get_dmem_size(pmu, in));
557 if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, in)))
558 goto clean_up;
559
560 if (payload->in.fb_size != 0x0) {
561 seq->in_mem = nvgpu_kzalloc(g,
562 sizeof(struct nvgpu_mem));
563 if (!seq->in_mem) {
564 err = -ENOMEM;
565 goto clean_up;
566 }
567
568 nvgpu_pmu_vidmem_surface_alloc(g, seq->in_mem,
569 payload->in.fb_size);
570 nvgpu_pmu_surface_describe(g, seq->in_mem,
571 (struct flcn_mem_desc_v0 *)
572 pv->pmu_allocation_get_fb_addr(pmu, in));
573
574 nvgpu_mem_wr_n(g, seq->in_mem, 0,
575 payload->in.buf, payload->in.fb_size);
576
577 } else {
578 nvgpu_flcn_copy_to_dmem(pmu->flcn,
579 (pv->pmu_allocation_get_dmem_offset(pmu, in)),
580 payload->in.buf, payload->in.size, 0);
581 }
582 pv->pmu_allocation_set_dmem_size(pmu,
583 pv->get_pmu_seq_in_a_ptr(seq),
584 pv->pmu_allocation_get_dmem_size(pmu, in));
585 pv->pmu_allocation_set_dmem_offset(pmu,
586 pv->get_pmu_seq_in_a_ptr(seq),
587 pv->pmu_allocation_get_dmem_offset(pmu, in));
588 }
589
590 if (payload && payload->out.offset != 0) {
591 pv->set_pmu_allocation_ptr(pmu, &out,
592 ((u8 *)&cmd->cmd + payload->out.offset));
593 pv->pmu_allocation_set_dmem_size(pmu, out,
594 (u16)payload->out.size);
595
596 if (payload->in.buf != payload->out.buf) {
597 *(pv->pmu_allocation_get_dmem_offset_addr(pmu, out)) =
598 nvgpu_alloc(&pmu->dmem,
599 pv->pmu_allocation_get_dmem_size(pmu, out));
600 if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu,
601 out)))
602 goto clean_up;
603
604 if (payload->out.fb_size != 0x0) {
605 seq->out_mem = nvgpu_kzalloc(g,
606 sizeof(struct nvgpu_mem));
607 if (!seq->out_mem) {
608 err = -ENOMEM;
609 goto clean_up;
610 }
611 nvgpu_pmu_vidmem_surface_alloc(g, seq->out_mem,
612 payload->out.fb_size);
613 nvgpu_pmu_surface_describe(g, seq->out_mem,
614 (struct flcn_mem_desc_v0 *)
615 pv->pmu_allocation_get_fb_addr(pmu,
616 out));
617 }
618 } else {
619 BUG_ON(in == NULL);
620 seq->out_mem = seq->in_mem;
621 pv->pmu_allocation_set_dmem_offset(pmu, out,
622 pv->pmu_allocation_get_dmem_offset(pmu, in));
623 }
624 pv->pmu_allocation_set_dmem_size(pmu,
625 pv->get_pmu_seq_out_a_ptr(seq),
626 pv->pmu_allocation_get_dmem_size(pmu, out));
627 pv->pmu_allocation_set_dmem_offset(pmu,
628 pv->get_pmu_seq_out_a_ptr(seq),
629 pv->pmu_allocation_get_dmem_offset(pmu, out));
630
631 }
632
633
634
635 seq->state = PMU_SEQ_STATE_USED;
636
637 err = pmu_write_cmd(pmu, cmd, queue_id, timeout);
638 if (err)
639 seq->state = PMU_SEQ_STATE_PENDING;
640
641 nvgpu_log_fn(g, "done");
642
643 return err;
644
645clean_up:
646 nvgpu_log_fn(g, "fail");
647 if (in)
648 nvgpu_free(&pmu->dmem,
649 pv->pmu_allocation_get_dmem_offset(pmu, in));
650 if (out)
651 nvgpu_free(&pmu->dmem,
652 pv->pmu_allocation_get_dmem_offset(pmu, out));
653
654 pmu_seq_release(pmu, seq);
655 return err;
656}
657
658static int pmu_response_handle(struct nvgpu_pmu *pmu,
659 struct pmu_msg *msg)
660{
661 struct gk20a *g = gk20a_from_pmu(pmu);
662 struct pmu_sequence *seq;
663 struct pmu_v *pv = &g->ops.pmu_ver;
664 int ret = 0;
665
666 nvgpu_log_fn(g, " ");
667
668 seq = &pmu->seq[msg->hdr.seq_id];
669 if (seq->state != PMU_SEQ_STATE_USED &&
670 seq->state != PMU_SEQ_STATE_CANCELLED) {
671 nvgpu_err(g, "msg for an unknown sequence %d", seq->id);
672 return -EINVAL;
673 }
674
675 if (msg->hdr.unit_id == PMU_UNIT_RC &&
676 msg->msg.rc.msg_type == PMU_RC_MSG_TYPE_UNHANDLED_CMD) {
677 nvgpu_err(g, "unhandled cmd: seq %d", seq->id);
678 } else if (seq->state != PMU_SEQ_STATE_CANCELLED) {
679 if (seq->msg) {
680 if (seq->msg->hdr.size >= msg->hdr.size) {
681 memcpy(seq->msg, msg, msg->hdr.size);
682 } else {
683 nvgpu_err(g, "sequence %d msg buffer too small",
684 seq->id);
685 }
686 }
687 if (pv->pmu_allocation_get_dmem_size(pmu,
688 pv->get_pmu_seq_out_a_ptr(seq)) != 0) {
689 nvgpu_flcn_copy_from_dmem(pmu->flcn,
690 pv->pmu_allocation_get_dmem_offset(pmu,
691 pv->get_pmu_seq_out_a_ptr(seq)),
692 seq->out_payload,
693 pv->pmu_allocation_get_dmem_size(pmu,
694 pv->get_pmu_seq_out_a_ptr(seq)), 0);
695 }
696 } else
697 seq->callback = NULL;
698 if (pv->pmu_allocation_get_dmem_size(pmu,
699 pv->get_pmu_seq_in_a_ptr(seq)) != 0)
700 nvgpu_free(&pmu->dmem,
701 pv->pmu_allocation_get_dmem_offset(pmu,
702 pv->get_pmu_seq_in_a_ptr(seq)));
703 if (pv->pmu_allocation_get_dmem_size(pmu,
704 pv->get_pmu_seq_out_a_ptr(seq)) != 0)
705 nvgpu_free(&pmu->dmem,
706 pv->pmu_allocation_get_dmem_offset(pmu,
707 pv->get_pmu_seq_out_a_ptr(seq)));
708
709 if (seq->out_mem != NULL) {
710 memset(pv->pmu_allocation_get_fb_addr(pmu,
711 pv->get_pmu_seq_out_a_ptr(seq)), 0x0,
712 pv->pmu_allocation_get_fb_size(pmu,
713 pv->get_pmu_seq_out_a_ptr(seq)));
714
715 nvgpu_pmu_surface_free(g, seq->out_mem);
716 if (seq->out_mem != seq->in_mem)
717 nvgpu_kfree(g, seq->out_mem);
718 else
719 seq->out_mem = NULL;
720 }
721
722 if (seq->in_mem != NULL) {
723 memset(pv->pmu_allocation_get_fb_addr(pmu,
724 pv->get_pmu_seq_in_a_ptr(seq)), 0x0,
725 pv->pmu_allocation_get_fb_size(pmu,
726 pv->get_pmu_seq_in_a_ptr(seq)));
727
728 nvgpu_pmu_surface_free(g, seq->in_mem);
729 nvgpu_kfree(g, seq->in_mem);
730 seq->in_mem = NULL;
731 }
732
733 if (seq->callback)
734 seq->callback(g, msg, seq->cb_params, seq->desc, ret);
735
736 pmu_seq_release(pmu, seq);
737
738 /* TBD: notify client waiting for available dmem */
739
740 nvgpu_log_fn(g, "done");
741
742 return 0;
743}
744
745static int pmu_handle_event(struct nvgpu_pmu *pmu, struct pmu_msg *msg)
746{
747 int err = 0;
748 struct gk20a *g = gk20a_from_pmu(pmu);
749
750 nvgpu_log_fn(g, " ");
751 switch (msg->hdr.unit_id) {
752 case PMU_UNIT_PERFMON:
753 case PMU_UNIT_PERFMON_T18X:
754 err = nvgpu_pmu_handle_perfmon_event(pmu, &msg->msg.perfmon);
755 break;
756 case PMU_UNIT_PERF:
757 if (g->ops.perf.handle_pmu_perf_event != NULL) {
758 err = g->ops.perf.handle_pmu_perf_event(g,
759 (void *)&msg->msg.perf);
760 } else {
761 WARN_ON(1);
762 }
763 break;
764 case PMU_UNIT_THERM:
765 err = nvgpu_pmu_handle_therm_event(pmu, &msg->msg.therm);
766 break;
767 default:
768 break;
769 }
770
771 return err;
772}
773
774static bool pmu_read_message(struct nvgpu_pmu *pmu, struct pmu_queue *queue,
775 struct pmu_msg *msg, int *status)
776{
777 struct gk20a *g = gk20a_from_pmu(pmu);
778 u32 read_size, bytes_read;
779 int err;
780
781 *status = 0;
782
783 if (nvgpu_pmu_queue_is_empty(pmu, queue))
784 return false;
785
786 err = pmu_queue_open_read(pmu, queue);
787 if (err) {
788 nvgpu_err(g, "fail to open queue %d for read", queue->id);
789 *status = err;
790 return false;
791 }
792
793 err = pmu_queue_pop(pmu, queue, &msg->hdr,
794 PMU_MSG_HDR_SIZE, &bytes_read);
795 if (err || bytes_read != PMU_MSG_HDR_SIZE) {
796 nvgpu_err(g, "fail to read msg from queue %d", queue->id);
797 *status = err | -EINVAL;
798 goto clean_up;
799 }
800
801 if (msg->hdr.unit_id == PMU_UNIT_REWIND) {
802 pmu_queue_rewind(pmu, queue);
803 /* read again after rewind */
804 err = pmu_queue_pop(pmu, queue, &msg->hdr,
805 PMU_MSG_HDR_SIZE, &bytes_read);
806 if (err || bytes_read != PMU_MSG_HDR_SIZE) {
807 nvgpu_err(g,
808 "fail to read msg from queue %d", queue->id);
809 *status = err | -EINVAL;
810 goto clean_up;
811 }
812 }
813
814 if (!PMU_UNIT_ID_IS_VALID(msg->hdr.unit_id)) {
815 nvgpu_err(g, "read invalid unit_id %d from queue %d",
816 msg->hdr.unit_id, queue->id);
817 *status = -EINVAL;
818 goto clean_up;
819 }
820
821 if (msg->hdr.size > PMU_MSG_HDR_SIZE) {
822 read_size = msg->hdr.size - PMU_MSG_HDR_SIZE;
823 err = pmu_queue_pop(pmu, queue, &msg->msg,
824 read_size, &bytes_read);
825 if (err || bytes_read != read_size) {
826 nvgpu_err(g,
827 "fail to read msg from queue %d", queue->id);
828 *status = err;
829 goto clean_up;
830 }
831 }
832
833 err = pmu_queue_close(pmu, queue, true);
834 if (err) {
835 nvgpu_err(g, "fail to close queue %d", queue->id);
836 *status = err;
837 return false;
838 }
839
840 return true;
841
842clean_up:
843 err = pmu_queue_close(pmu, queue, false);
844 if (err)
845 nvgpu_err(g, "fail to close queue %d", queue->id);
846 return false;
847}
848
849int nvgpu_pmu_process_message(struct nvgpu_pmu *pmu)
850{
851 struct pmu_msg msg;
852 int status;
853 struct gk20a *g = gk20a_from_pmu(pmu);
854
855 if (unlikely(!pmu->pmu_ready)) {
856 nvgpu_pmu_process_init_msg(pmu, &msg);
857 if (g->ops.pmu.init_wpr_region != NULL)
858 g->ops.pmu.init_wpr_region(g);
859 if (nvgpu_is_enabled(g, NVGPU_PMU_PERFMON))
860 nvgpu_pmu_init_perfmon(pmu);
861
862 return 0;
863 }
864
865 while (pmu_read_message(pmu,
866 &pmu->queue[PMU_MESSAGE_QUEUE], &msg, &status)) {
867
868 nvgpu_pmu_dbg(g, "read msg hdr: ");
869 nvgpu_pmu_dbg(g, "unit_id = 0x%08x, size = 0x%08x",
870 msg.hdr.unit_id, msg.hdr.size);
871 nvgpu_pmu_dbg(g, "ctrl_flags = 0x%08x, seq_id = 0x%08x",
872 msg.hdr.ctrl_flags, msg.hdr.seq_id);
873
874 msg.hdr.ctrl_flags &= ~PMU_CMD_FLAGS_PMU_MASK;
875
876 if (msg.hdr.ctrl_flags == PMU_CMD_FLAGS_EVENT)
877 pmu_handle_event(pmu, &msg);
878 else
879 pmu_response_handle(pmu, &msg);
880 }
881
882 return 0;
883}
884
885int pmu_wait_message_cond(struct nvgpu_pmu *pmu, u32 timeout_ms,
886 u32 *var, u32 val)
887{
888 struct gk20a *g = gk20a_from_pmu(pmu);
889 struct nvgpu_timeout timeout;
890 unsigned long delay = GR_IDLE_CHECK_DEFAULT;
891
892 nvgpu_timeout_init(g, &timeout, timeout_ms, NVGPU_TIMER_CPU_TIMER);
893
894 do {
895 if (*var == val)
896 return 0;
897
898 if (gk20a_pmu_is_interrupted(pmu))
899 gk20a_pmu_isr(g);
900
901 nvgpu_usleep_range(delay, delay * 2);
902 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
903 } while (!nvgpu_timeout_expired(&timeout));
904
905 return -ETIMEDOUT;
906}
907
diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_perfmon.c b/drivers/gpu/nvgpu/common/pmu/pmu_perfmon.c
new file mode 100644
index 00000000..2b952868
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/pmu/pmu_perfmon.c
@@ -0,0 +1,293 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/enabled.h>
24#include <nvgpu/pmu.h>
25#include <nvgpu/log.h>
26#include <nvgpu/bug.h>
27#include <nvgpu/pmuif/nvgpu_gpmu_cmdif.h>
28
29#include "gk20a/gk20a.h"
30
31#ifdef CONFIG_TEGRA_19x_GPU
32#include "nvgpu_gpuid_t19x.h"
33#endif
34
35static u8 get_perfmon_id(struct nvgpu_pmu *pmu)
36{
37 struct gk20a *g = gk20a_from_pmu(pmu);
38 u32 ver = g->params.gpu_arch + g->params.gpu_impl;
39 u8 unit_id;
40
41 switch (ver) {
42 case GK20A_GPUID_GK20A:
43 case GK20A_GPUID_GM20B:
44 case GK20A_GPUID_GM20B_B:
45 unit_id = PMU_UNIT_PERFMON;
46 break;
47 case NVGPU_GPUID_GP10B:
48 case NVGPU_GPUID_GP104:
49 case NVGPU_GPUID_GP106:
50 unit_id = PMU_UNIT_PERFMON_T18X;
51 break;
52#if defined(CONFIG_TEGRA_19x_GPU)
53 case TEGRA_19x_GPUID:
54 unit_id = PMU_UNIT_PERFMON_T18X;
55 break;
56#endif
57 default:
58 unit_id = PMU_UNIT_INVALID;
59 nvgpu_err(g, "no support for %x", ver);
60 WARN_ON(1);
61 }
62
63 return unit_id;
64}
65
66int nvgpu_pmu_init_perfmon(struct nvgpu_pmu *pmu)
67{
68 struct gk20a *g = gk20a_from_pmu(pmu);
69 struct pmu_v *pv = &g->ops.pmu_ver;
70 struct pmu_cmd cmd;
71 struct pmu_payload payload;
72 u32 seq;
73
74 if (!nvgpu_is_enabled(g, NVGPU_PMU_PERFMON))
75 return 0;
76
77 nvgpu_log_fn(g, " ");
78
79 pmu->perfmon_ready = 0;
80
81 gk20a_pmu_init_perfmon_counter(g);
82
83 if (!pmu->sample_buffer)
84 pmu->sample_buffer = nvgpu_alloc(&pmu->dmem,
85 2 * sizeof(u16));
86 if (!pmu->sample_buffer) {
87 nvgpu_err(g, "failed to allocate perfmon sample buffer");
88 return -ENOMEM;
89 }
90
91 /* init PERFMON */
92 memset(&cmd, 0, sizeof(struct pmu_cmd));
93
94 cmd.hdr.unit_id = get_perfmon_id(pmu);
95 if (cmd.hdr.unit_id == PMU_UNIT_INVALID) {
96 nvgpu_err(g, "failed to get perfmon UNIT ID, command skipped");
97 return -EINVAL;
98 }
99
100 cmd.hdr.size = PMU_CMD_HDR_SIZE + pv->get_pmu_perfmon_cmd_init_size();
101 cmd.cmd.perfmon.cmd_type = PMU_PERFMON_CMD_ID_INIT;
102 /* buffer to save counter values for pmu perfmon */
103 pv->perfmon_cmd_init_set_sample_buffer(&cmd.cmd.perfmon,
104 (u16)pmu->sample_buffer);
105 /* number of sample periods below lower threshold
106 * before pmu triggers perfmon decrease event
107 * TBD: = 15
108 */
109 pv->perfmon_cmd_init_set_dec_cnt(&cmd.cmd.perfmon, 15);
110 /* index of base counter, aka. always ticking counter */
111 pv->perfmon_cmd_init_set_base_cnt_id(&cmd.cmd.perfmon, 6);
112 /* microseconds interval between pmu polls perf counters */
113 pv->perfmon_cmd_init_set_samp_period_us(&cmd.cmd.perfmon, 16700);
114 /* number of perfmon counters
115 * counter #3 (GR and CE2) for gk20a
116 */
117 pv->perfmon_cmd_init_set_num_cnt(&cmd.cmd.perfmon, 1);
118 /* moving average window for sample periods
119 * TBD: = 3000000 / sample_period_us = 17
120 */
121 pv->perfmon_cmd_init_set_mov_avg(&cmd.cmd.perfmon, 17);
122
123 memset(&payload, 0, sizeof(struct pmu_payload));
124 payload.in.buf = pv->get_perfmon_cntr_ptr(pmu);
125 payload.in.size = pv->get_perfmon_cntr_sz(pmu);
126 payload.in.offset = pv->get_perfmon_cmd_init_offsetofvar(COUNTER_ALLOC);
127
128 nvgpu_pmu_dbg(g, "cmd post PMU_PERFMON_CMD_ID_INIT");
129 nvgpu_pmu_cmd_post(g, &cmd, NULL, &payload, PMU_COMMAND_QUEUE_LPQ,
130 NULL, NULL, &seq, ~0);
131
132 return 0;
133}
134
135int nvgpu_pmu_perfmon_start_sampling(struct nvgpu_pmu *pmu)
136{
137 struct gk20a *g = gk20a_from_pmu(pmu);
138 struct pmu_v *pv = &g->ops.pmu_ver;
139 struct pmu_cmd cmd;
140 struct pmu_payload payload;
141 u32 seq;
142
143 if (!nvgpu_is_enabled(g, NVGPU_PMU_PERFMON))
144 return 0;
145
146 /* PERFMON Start */
147 memset(&cmd, 0, sizeof(struct pmu_cmd));
148 cmd.hdr.unit_id = get_perfmon_id(pmu);
149 if (cmd.hdr.unit_id == PMU_UNIT_INVALID) {
150 nvgpu_err(g, "failed to get perfmon UNIT ID, command skipped");
151 return -EINVAL;
152 }
153 cmd.hdr.size = PMU_CMD_HDR_SIZE + pv->get_pmu_perfmon_cmd_start_size();
154 pv->perfmon_start_set_cmd_type(&cmd.cmd.perfmon,
155 PMU_PERFMON_CMD_ID_START);
156 pv->perfmon_start_set_group_id(&cmd.cmd.perfmon,
157 PMU_DOMAIN_GROUP_PSTATE);
158 pv->perfmon_start_set_state_id(&cmd.cmd.perfmon,
159 pmu->perfmon_state_id[PMU_DOMAIN_GROUP_PSTATE]);
160
161 pv->perfmon_start_set_flags(&cmd.cmd.perfmon,
162 PMU_PERFMON_FLAG_ENABLE_INCREASE |
163 PMU_PERFMON_FLAG_ENABLE_DECREASE |
164 PMU_PERFMON_FLAG_CLEAR_PREV);
165
166 memset(&payload, 0, sizeof(struct pmu_payload));
167
168 /* TBD: PMU_PERFMON_PCT_TO_INC * 100 */
169 pv->set_perfmon_cntr_ut(pmu, 3000); /* 30% */
170 /* TBD: PMU_PERFMON_PCT_TO_DEC * 100 */
171 pv->set_perfmon_cntr_lt(pmu, 1000); /* 10% */
172 pv->set_perfmon_cntr_valid(pmu, true);
173
174 payload.in.buf = pv->get_perfmon_cntr_ptr(pmu);
175 payload.in.size = pv->get_perfmon_cntr_sz(pmu);
176 payload.in.offset =
177 pv->get_perfmon_cmd_start_offsetofvar(COUNTER_ALLOC);
178
179 nvgpu_pmu_dbg(g, "cmd post PMU_PERFMON_CMD_ID_START");
180 nvgpu_pmu_cmd_post(g, &cmd, NULL, &payload, PMU_COMMAND_QUEUE_LPQ,
181 NULL, NULL, &seq, ~0);
182
183 return 0;
184}
185
186int nvgpu_pmu_perfmon_stop_sampling(struct nvgpu_pmu *pmu)
187{
188 struct gk20a *g = gk20a_from_pmu(pmu);
189 struct pmu_cmd cmd;
190 u32 seq;
191
192 if (!nvgpu_is_enabled(g, NVGPU_PMU_PERFMON))
193 return 0;
194
195 /* PERFMON Stop */
196 memset(&cmd, 0, sizeof(struct pmu_cmd));
197 cmd.hdr.unit_id = get_perfmon_id(pmu);
198 if (cmd.hdr.unit_id == PMU_UNIT_INVALID) {
199 nvgpu_err(g, "failed to get perfmon UNIT ID, command skipped");
200 return -EINVAL;
201 }
202 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_perfmon_cmd_stop);
203 cmd.cmd.perfmon.stop.cmd_type = PMU_PERFMON_CMD_ID_STOP;
204
205 nvgpu_pmu_dbg(g, "cmd post PMU_PERFMON_CMD_ID_STOP");
206 nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
207 NULL, NULL, &seq, ~0);
208 return 0;
209}
210
211int nvgpu_pmu_load_norm(struct gk20a *g, u32 *load)
212{
213 *load = g->pmu.load_shadow;
214 return 0;
215}
216
217int nvgpu_pmu_load_update(struct gk20a *g)
218{
219 struct nvgpu_pmu *pmu = &g->pmu;
220 u16 load = 0;
221
222 if (!pmu->perfmon_ready) {
223 pmu->load_shadow = 0;
224 return 0;
225 }
226
227 nvgpu_flcn_copy_from_dmem(pmu->flcn, pmu->sample_buffer,
228 (u8 *)&load, 2, 0);
229 pmu->load_shadow = load / 10;
230 pmu->load_avg = (((9*pmu->load_avg) + pmu->load_shadow) / 10);
231
232 return 0;
233}
234
235void nvgpu_pmu_get_load_counters(struct gk20a *g, u32 *busy_cycles,
236 u32 *total_cycles)
237{
238 if (!g->power_on || gk20a_busy(g)) {
239 *busy_cycles = 0;
240 *total_cycles = 0;
241 return;
242 }
243
244 *busy_cycles = gk20a_pmu_read_idle_counter(g, 1);
245 *total_cycles = gk20a_pmu_read_idle_counter(g, 2);
246
247 gk20a_idle(g);
248}
249
250void nvgpu_pmu_reset_load_counters(struct gk20a *g)
251{
252 if (!g->power_on || gk20a_busy(g))
253 return;
254
255 gk20a_pmu_reset_idle_counter(g, 2);
256 gk20a_pmu_reset_idle_counter(g, 1);
257
258 gk20a_idle(g);
259}
260
261int nvgpu_pmu_handle_perfmon_event(struct nvgpu_pmu *pmu,
262 struct pmu_perfmon_msg *msg)
263{
264 struct gk20a *g = gk20a_from_pmu(pmu);
265
266 nvgpu_log_fn(g, " ");
267
268 switch (msg->msg_type) {
269 case PMU_PERFMON_MSG_ID_INCREASE_EVENT:
270 nvgpu_pmu_dbg(g, "perfmon increase event: ");
271 nvgpu_pmu_dbg(g, "state_id %d, ground_id %d, pct %d",
272 msg->gen.state_id, msg->gen.group_id, msg->gen.data);
273 (pmu->perfmon_events_cnt)++;
274 break;
275 case PMU_PERFMON_MSG_ID_DECREASE_EVENT:
276 nvgpu_pmu_dbg(g, "perfmon decrease event: ");
277 nvgpu_pmu_dbg(g, "state_id %d, ground_id %d, pct %d",
278 msg->gen.state_id, msg->gen.group_id, msg->gen.data);
279 (pmu->perfmon_events_cnt)++;
280 break;
281 case PMU_PERFMON_MSG_ID_INIT_EVENT:
282 pmu->perfmon_ready = 1;
283 nvgpu_pmu_dbg(g, "perfmon init event");
284 break;
285 default:
286 break;
287 }
288
289 /* restart sampling */
290 if (pmu->perfmon_sampling_enabled)
291 return nvgpu_pmu_perfmon_start_sampling(pmu);
292 return 0;
293}
diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_pg.c b/drivers/gpu/nvgpu/common/pmu/pmu_pg.c
new file mode 100644
index 00000000..bf39ce19
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/pmu/pmu_pg.c
@@ -0,0 +1,748 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/pmu.h>
24#include <nvgpu/log.h>
25#include <nvgpu/pmuif/nvgpu_gpmu_cmdif.h>
26#include <nvgpu/barrier.h>
27#include <nvgpu/bug.h>
28
29#include "gk20a/gk20a.h"
30
31/* state transition :
32 * OFF => [OFF_ON_PENDING optional] => ON_PENDING => ON => OFF
33 * ON => OFF is always synchronized
34 */
35/* elpg is off */
36#define PMU_ELPG_STAT_OFF 0
37/* elpg is on */
38#define PMU_ELPG_STAT_ON 1
39/* elpg is off, ALLOW cmd has been sent, wait for ack */
40#define PMU_ELPG_STAT_ON_PENDING 2
41/* elpg is on, DISALLOW cmd has been sent, wait for ack */
42#define PMU_ELPG_STAT_OFF_PENDING 3
43/* elpg is off, caller has requested on, but ALLOW
44 * cmd hasn't been sent due to ENABLE_ALLOW delay
45 */
46#define PMU_ELPG_STAT_OFF_ON_PENDING 4
47
48#define PMU_PGENG_GR_BUFFER_IDX_INIT (0)
49#define PMU_PGENG_GR_BUFFER_IDX_ZBC (1)
50#define PMU_PGENG_GR_BUFFER_IDX_FECS (2)
51
52static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
53 void *param, u32 handle, u32 status)
54{
55 struct nvgpu_pmu *pmu = param;
56 struct pmu_pg_msg_elpg_msg *elpg_msg = &msg->msg.pg.elpg_msg;
57
58 nvgpu_log_fn(g, " ");
59
60 if (status != 0) {
61 nvgpu_err(g, "ELPG cmd aborted");
62 /* TBD: disable ELPG */
63 return;
64 }
65
66 switch (elpg_msg->msg) {
67 case PMU_PG_ELPG_MSG_INIT_ACK:
68 nvgpu_pmu_dbg(g, "INIT_PG is ack from PMU, eng - %d",
69 elpg_msg->engine_id);
70 break;
71 case PMU_PG_ELPG_MSG_ALLOW_ACK:
72 nvgpu_pmu_dbg(g, "ALLOW is ack from PMU, eng - %d",
73 elpg_msg->engine_id);
74 if (elpg_msg->engine_id == PMU_PG_ELPG_ENGINE_ID_MS)
75 pmu->mscg_transition_state = PMU_ELPG_STAT_ON;
76 else
77 pmu->elpg_stat = PMU_ELPG_STAT_ON;
78 break;
79 case PMU_PG_ELPG_MSG_DISALLOW_ACK:
80 nvgpu_pmu_dbg(g, "DISALLOW is ack from PMU, eng - %d",
81 elpg_msg->engine_id);
82
83 if (elpg_msg->engine_id == PMU_PG_ELPG_ENGINE_ID_MS)
84 pmu->mscg_transition_state = PMU_ELPG_STAT_OFF;
85 else
86 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
87
88 if (pmu->pmu_state == PMU_STATE_ELPG_BOOTING) {
89 if (g->ops.pmu.pmu_pg_engines_feature_list &&
90 g->ops.pmu.pmu_pg_engines_feature_list(g,
91 PMU_PG_ELPG_ENGINE_ID_GRAPHICS) !=
92 PMU_PG_FEATURE_GR_POWER_GATING_ENABLED) {
93 pmu->initialized = true;
94 nvgpu_pmu_state_change(g, PMU_STATE_STARTED,
95 true);
96 WRITE_ONCE(pmu->mscg_stat, PMU_MSCG_DISABLED);
97 /* make status visible */
98 nvgpu_smp_mb();
99 } else
100 nvgpu_pmu_state_change(g, PMU_STATE_ELPG_BOOTED,
101 true);
102 }
103 break;
104 default:
105 nvgpu_err(g,
106 "unsupported ELPG message : 0x%04x", elpg_msg->msg);
107 }
108}
109
110/* PG enable/disable */
111int nvgpu_pmu_pg_global_enable(struct gk20a *g, u32 enable_pg)
112{
113 u32 status = 0;
114
115 if (enable_pg == true) {
116 if (g->ops.pmu.pmu_pg_engines_feature_list &&
117 g->ops.pmu.pmu_pg_engines_feature_list(g,
118 PMU_PG_ELPG_ENGINE_ID_GRAPHICS) !=
119 PMU_PG_FEATURE_GR_POWER_GATING_ENABLED) {
120 if (g->ops.pmu.pmu_lpwr_enable_pg)
121 status = g->ops.pmu.pmu_lpwr_enable_pg(g,
122 true);
123 } else if (g->support_pmu && g->can_elpg)
124 status = nvgpu_pmu_enable_elpg(g);
125 } else if (enable_pg == false) {
126 if (g->ops.pmu.pmu_pg_engines_feature_list &&
127 g->ops.pmu.pmu_pg_engines_feature_list(g,
128 PMU_PG_ELPG_ENGINE_ID_GRAPHICS) !=
129 PMU_PG_FEATURE_GR_POWER_GATING_ENABLED) {
130 if (g->ops.pmu.pmu_lpwr_disable_pg)
131 status = g->ops.pmu.pmu_lpwr_disable_pg(g,
132 true);
133 } else if (g->support_pmu && g->can_elpg)
134 status = nvgpu_pmu_disable_elpg(g);
135 }
136
137 return status;
138}
139
140static int pmu_enable_elpg_locked(struct gk20a *g, u32 pg_engine_id)
141{
142 struct nvgpu_pmu *pmu = &g->pmu;
143 struct pmu_cmd cmd;
144 u32 seq, status;
145
146 nvgpu_log_fn(g, " ");
147
148 memset(&cmd, 0, sizeof(struct pmu_cmd));
149 cmd.hdr.unit_id = PMU_UNIT_PG;
150 cmd.hdr.size = PMU_CMD_HDR_SIZE +
151 sizeof(struct pmu_pg_cmd_elpg_cmd);
152 cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
153 cmd.cmd.pg.elpg_cmd.engine_id = pg_engine_id;
154 cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_ALLOW;
155
156 /* no need to wait ack for ELPG enable but set
157 * pending to sync with follow up ELPG disable
158 */
159 if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS)
160 pmu->elpg_stat = PMU_ELPG_STAT_ON_PENDING;
161 else if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS)
162 pmu->mscg_transition_state = PMU_ELPG_STAT_ON_PENDING;
163
164 nvgpu_pmu_dbg(g, "cmd post PMU_PG_ELPG_CMD_ALLOW");
165 status = nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL,
166 PMU_COMMAND_QUEUE_HPQ, pmu_handle_pg_elpg_msg,
167 pmu, &seq, ~0);
168 WARN_ON(status != 0);
169
170 nvgpu_log_fn(g, "done");
171 return 0;
172}
173
174int nvgpu_pmu_enable_elpg(struct gk20a *g)
175{
176 struct nvgpu_pmu *pmu = &g->pmu;
177 struct gr_gk20a *gr = &g->gr;
178 u32 pg_engine_id;
179 u32 pg_engine_id_list = 0;
180
181 int ret = 0;
182
183 nvgpu_log_fn(g, " ");
184
185 if (!g->support_pmu)
186 return ret;
187
188 nvgpu_mutex_acquire(&pmu->elpg_mutex);
189
190 pmu->elpg_refcnt++;
191 if (pmu->elpg_refcnt <= 0)
192 goto exit_unlock;
193
194 /* something is not right if we end up in following code path */
195 if (unlikely(pmu->elpg_refcnt > 1)) {
196 nvgpu_warn(g,
197 "%s(): possible elpg refcnt mismatch. elpg refcnt=%d",
198 __func__, pmu->elpg_refcnt);
199 WARN_ON(1);
200 }
201
202 /* do NOT enable elpg until golden ctx is created,
203 * which is related with the ctx that ELPG save and restore.
204 */
205 if (unlikely(!gr->ctx_vars.golden_image_initialized))
206 goto exit_unlock;
207
208 /* return if ELPG is already on or on_pending or off_on_pending */
209 if (pmu->elpg_stat != PMU_ELPG_STAT_OFF)
210 goto exit_unlock;
211
212 if (g->ops.pmu.pmu_pg_supported_engines_list)
213 pg_engine_id_list = g->ops.pmu.pmu_pg_supported_engines_list(g);
214
215 for (pg_engine_id = PMU_PG_ELPG_ENGINE_ID_GRAPHICS;
216 pg_engine_id < PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE;
217 pg_engine_id++) {
218
219 if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS &&
220 pmu->mscg_stat == PMU_MSCG_DISABLED)
221 continue;
222
223 if (BIT(pg_engine_id) & pg_engine_id_list)
224 ret = pmu_enable_elpg_locked(g, pg_engine_id);
225 }
226
227exit_unlock:
228 nvgpu_mutex_release(&pmu->elpg_mutex);
229 nvgpu_log_fn(g, "done");
230 return ret;
231}
232
233int nvgpu_pmu_disable_elpg(struct gk20a *g)
234{
235 struct nvgpu_pmu *pmu = &g->pmu;
236 struct pmu_cmd cmd;
237 u32 seq;
238 int ret = 0;
239 u32 pg_engine_id;
240 u32 pg_engine_id_list = 0;
241 u32 *ptr = NULL;
242
243 nvgpu_log_fn(g, " ");
244
245 if (g->ops.pmu.pmu_pg_supported_engines_list)
246 pg_engine_id_list = g->ops.pmu.pmu_pg_supported_engines_list(g);
247
248 if (!g->support_pmu)
249 return ret;
250
251 nvgpu_mutex_acquire(&pmu->elpg_mutex);
252
253 pmu->elpg_refcnt--;
254 if (pmu->elpg_refcnt > 0) {
255 nvgpu_warn(g,
256 "%s(): possible elpg refcnt mismatch. elpg refcnt=%d",
257 __func__, pmu->elpg_refcnt);
258 WARN_ON(1);
259 ret = 0;
260 goto exit_unlock;
261 }
262
263 /* cancel off_on_pending and return */
264 if (pmu->elpg_stat == PMU_ELPG_STAT_OFF_ON_PENDING) {
265 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
266 ret = 0;
267 goto exit_reschedule;
268 }
269 /* wait if on_pending */
270 else if (pmu->elpg_stat == PMU_ELPG_STAT_ON_PENDING) {
271
272 pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
273 &pmu->elpg_stat, PMU_ELPG_STAT_ON);
274
275 if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
276 nvgpu_err(g, "ELPG_ALLOW_ACK failed, elpg_stat=%d",
277 pmu->elpg_stat);
278 nvgpu_pmu_dump_elpg_stats(pmu);
279 nvgpu_pmu_dump_falcon_stats(pmu);
280 ret = -EBUSY;
281 goto exit_unlock;
282 }
283 }
284 /* return if ELPG is already off */
285 else if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
286 ret = 0;
287 goto exit_reschedule;
288 }
289
290 for (pg_engine_id = PMU_PG_ELPG_ENGINE_ID_GRAPHICS;
291 pg_engine_id < PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE;
292 pg_engine_id++) {
293
294 if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS &&
295 pmu->mscg_stat == PMU_MSCG_DISABLED)
296 continue;
297
298 if (BIT(pg_engine_id) & pg_engine_id_list) {
299 memset(&cmd, 0, sizeof(struct pmu_cmd));
300 cmd.hdr.unit_id = PMU_UNIT_PG;
301 cmd.hdr.size = PMU_CMD_HDR_SIZE +
302 sizeof(struct pmu_pg_cmd_elpg_cmd);
303 cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
304 cmd.cmd.pg.elpg_cmd.engine_id = pg_engine_id;
305 cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
306
307 if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS)
308 pmu->elpg_stat = PMU_ELPG_STAT_OFF_PENDING;
309 else if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS)
310 pmu->mscg_transition_state =
311 PMU_ELPG_STAT_OFF_PENDING;
312
313 if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS)
314 ptr = &pmu->elpg_stat;
315 else if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS)
316 ptr = &pmu->mscg_transition_state;
317
318 nvgpu_pmu_dbg(g, "cmd post PMU_PG_ELPG_CMD_DISALLOW");
319 nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL,
320 PMU_COMMAND_QUEUE_HPQ, pmu_handle_pg_elpg_msg,
321 pmu, &seq, ~0);
322
323 pmu_wait_message_cond(pmu,
324 gk20a_get_gr_idle_timeout(g),
325 ptr, PMU_ELPG_STAT_OFF);
326 if (*ptr != PMU_ELPG_STAT_OFF) {
327 nvgpu_err(g, "ELPG_DISALLOW_ACK failed");
328 nvgpu_pmu_dump_elpg_stats(pmu);
329 nvgpu_pmu_dump_falcon_stats(pmu);
330 ret = -EBUSY;
331 goto exit_unlock;
332 }
333 }
334 }
335
336exit_reschedule:
337exit_unlock:
338 nvgpu_mutex_release(&pmu->elpg_mutex);
339 nvgpu_log_fn(g, "done");
340 return ret;
341}
342
343/* PG init */
344static void pmu_handle_pg_stat_msg(struct gk20a *g, struct pmu_msg *msg,
345 void *param, u32 handle, u32 status)
346{
347 struct nvgpu_pmu *pmu = param;
348
349 nvgpu_log_fn(g, " ");
350
351 if (status != 0) {
352 nvgpu_err(g, "ELPG cmd aborted");
353 /* TBD: disable ELPG */
354 return;
355 }
356
357 switch (msg->msg.pg.stat.sub_msg_id) {
358 case PMU_PG_STAT_MSG_RESP_DMEM_OFFSET:
359 nvgpu_pmu_dbg(g, "ALLOC_DMEM_OFFSET is acknowledged from PMU");
360 pmu->stat_dmem_offset[msg->msg.pg.stat.engine_id] =
361 msg->msg.pg.stat.data;
362 break;
363 default:
364 break;
365 }
366}
367
368static int pmu_pg_init_send(struct gk20a *g, u32 pg_engine_id)
369{
370 struct nvgpu_pmu *pmu = &g->pmu;
371 struct pmu_cmd cmd;
372 u32 seq;
373 int err = 0;
374
375 nvgpu_log_fn(g, " ");
376
377 if (pmu->pmu_state == PMU_STATE_INIT_RECEIVED)
378 nvgpu_pmu_state_change(g,
379 PMU_STATE_ELPG_BOOTING, false);
380 else
381 nvgpu_err(g, "PMU INIT not received\n");
382
383 gk20a_pmu_pg_idle_counter_config(g, pg_engine_id);
384
385 if (g->ops.pmu.pmu_pg_init_param)
386 g->ops.pmu.pmu_pg_init_param(g, pg_engine_id);
387
388 /* init ELPG */
389 memset(&cmd, 0, sizeof(struct pmu_cmd));
390 cmd.hdr.unit_id = PMU_UNIT_PG;
391 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
392 cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
393 cmd.cmd.pg.elpg_cmd.engine_id = pg_engine_id;
394 cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_INIT;
395
396 nvgpu_pmu_dbg(g, "cmd post PMU_PG_ELPG_CMD_INIT");
397 err = nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
398 pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
399 if (err)
400 nvgpu_err(g, "PMU_PG_ELPG_CMD_INIT cmd failed\n");
401
402 /* alloc dmem for powergating state log */
403 pmu->stat_dmem_offset[pg_engine_id] = 0;
404 memset(&cmd, 0, sizeof(struct pmu_cmd));
405 cmd.hdr.unit_id = PMU_UNIT_PG;
406 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_stat);
407 cmd.cmd.pg.stat.cmd_type = PMU_PG_CMD_ID_PG_STAT;
408 cmd.cmd.pg.stat.engine_id = pg_engine_id;
409 cmd.cmd.pg.stat.sub_cmd_id = PMU_PG_STAT_CMD_ALLOC_DMEM;
410 cmd.cmd.pg.stat.data = 0;
411
412 nvgpu_pmu_dbg(g, "cmd post PMU_PG_STAT_CMD_ALLOC_DMEM");
413 err = nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
414 pmu_handle_pg_stat_msg, pmu, &seq, ~0);
415 if (err)
416 nvgpu_err(g, "PMU_PG_STAT_CMD_ALLOC_DMEM cmd failed\n");
417
418 /* disallow ELPG initially
419 * PMU ucode requires a disallow cmd before allow cmd
420 */
421 /* set for wait_event PMU_ELPG_STAT_OFF */
422 if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS)
423 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
424 else if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS)
425 pmu->mscg_transition_state = PMU_ELPG_STAT_OFF;
426 memset(&cmd, 0, sizeof(struct pmu_cmd));
427 cmd.hdr.unit_id = PMU_UNIT_PG;
428 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
429 cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
430 cmd.cmd.pg.elpg_cmd.engine_id = pg_engine_id;
431 cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
432
433 nvgpu_pmu_dbg(g, "cmd post PMU_PG_ELPG_CMD_DISALLOW");
434 err = nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
435 pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
436 if (err)
437 nvgpu_err(g, "PMU_PG_ELPG_CMD_DISALLOW cmd failed\n");
438
439 if (g->ops.pmu.pmu_pg_set_sub_feature_mask)
440 g->ops.pmu.pmu_pg_set_sub_feature_mask(g, pg_engine_id);
441
442 return 0;
443}
444
445int nvgpu_pmu_init_powergating(struct gk20a *g)
446{
447 u32 pg_engine_id;
448 u32 pg_engine_id_list = 0;
449
450 nvgpu_log_fn(g, " ");
451
452 if (g->ops.pmu.pmu_pg_supported_engines_list)
453 pg_engine_id_list = g->ops.pmu.pmu_pg_supported_engines_list(g);
454
455 gk20a_gr_wait_initialized(g);
456
457 for (pg_engine_id = PMU_PG_ELPG_ENGINE_ID_GRAPHICS;
458 pg_engine_id < PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE;
459 pg_engine_id++) {
460
461 if (BIT(pg_engine_id) & pg_engine_id_list) {
462 pmu_pg_init_send(g, pg_engine_id);
463 }
464 }
465
466 if (g->ops.pmu.pmu_pg_param_post_init)
467 g->ops.pmu.pmu_pg_param_post_init(g);
468
469 return 0;
470}
471
472static void pmu_handle_pg_buf_config_msg(struct gk20a *g, struct pmu_msg *msg,
473 void *param, u32 handle, u32 status)
474{
475 struct nvgpu_pmu *pmu = param;
476 struct pmu_pg_msg_eng_buf_stat *eng_buf_stat =
477 &msg->msg.pg.eng_buf_stat;
478
479 nvgpu_log_fn(g, " ");
480
481 nvgpu_pmu_dbg(g,
482 "reply PMU_PG_CMD_ID_ENG_BUF_LOAD PMU_PGENG_GR_BUFFER_IDX_FECS");
483 if (status != 0) {
484 nvgpu_err(g, "PGENG cmd aborted");
485 /* TBD: disable ELPG */
486 return;
487 }
488
489 pmu->buf_loaded = (eng_buf_stat->status == PMU_PG_MSG_ENG_BUF_LOADED);
490 if ((!pmu->buf_loaded) &&
491 (pmu->pmu_state == PMU_STATE_LOADING_PG_BUF))
492 nvgpu_err(g, "failed to load PGENG buffer");
493 else {
494 nvgpu_pmu_state_change(g, pmu->pmu_state, true);
495 }
496}
497
498int nvgpu_pmu_init_bind_fecs(struct gk20a *g)
499{
500 struct nvgpu_pmu *pmu = &g->pmu;
501 struct pmu_cmd cmd;
502 u32 desc;
503 int err = 0;
504 u32 gr_engine_id;
505
506 nvgpu_log_fn(g, " ");
507
508 gr_engine_id = gk20a_fifo_get_gr_engine_id(g);
509
510 memset(&cmd, 0, sizeof(struct pmu_cmd));
511 cmd.hdr.unit_id = PMU_UNIT_PG;
512 cmd.hdr.size = PMU_CMD_HDR_SIZE +
513 g->ops.pmu_ver.pg_cmd_eng_buf_load_size(&cmd.cmd.pg);
514 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_cmd_type(&cmd.cmd.pg,
515 PMU_PG_CMD_ID_ENG_BUF_LOAD);
516 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_engine_id(&cmd.cmd.pg,
517 gr_engine_id);
518 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_idx(&cmd.cmd.pg,
519 PMU_PGENG_GR_BUFFER_IDX_FECS);
520 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_size(&cmd.cmd.pg,
521 pmu->pg_buf.size);
522 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_base(&cmd.cmd.pg,
523 u64_lo32(pmu->pg_buf.gpu_va));
524 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_offset(&cmd.cmd.pg,
525 (u8)(pmu->pg_buf.gpu_va & 0xFF));
526 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_idx(&cmd.cmd.pg,
527 PMU_DMAIDX_VIRT);
528
529 pmu->buf_loaded = false;
530 nvgpu_pmu_dbg(g, "cmd post PMU_PG_CMD_ID_ENG_BUF_LOAD PMU_PGENG_GR_BUFFER_IDX_FECS");
531 nvgpu_pmu_state_change(g, PMU_STATE_LOADING_PG_BUF, false);
532 err = nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
533 pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
534 if (err)
535 nvgpu_err(g, "cmd LOAD PMU_PGENG_GR_BUFFER_IDX_FECS failed\n");
536
537 return err;
538}
539
540void nvgpu_pmu_setup_hw_load_zbc(struct gk20a *g)
541{
542 struct nvgpu_pmu *pmu = &g->pmu;
543 struct pmu_cmd cmd;
544 u32 desc;
545 u32 gr_engine_id;
546 int err = 0;
547
548 gr_engine_id = gk20a_fifo_get_gr_engine_id(g);
549
550 memset(&cmd, 0, sizeof(struct pmu_cmd));
551 cmd.hdr.unit_id = PMU_UNIT_PG;
552 cmd.hdr.size = PMU_CMD_HDR_SIZE +
553 g->ops.pmu_ver.pg_cmd_eng_buf_load_size(&cmd.cmd.pg);
554 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_cmd_type(&cmd.cmd.pg,
555 PMU_PG_CMD_ID_ENG_BUF_LOAD);
556 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_engine_id(&cmd.cmd.pg,
557 gr_engine_id);
558 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_idx(&cmd.cmd.pg,
559 PMU_PGENG_GR_BUFFER_IDX_ZBC);
560 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_buf_size(&cmd.cmd.pg,
561 pmu->seq_buf.size);
562 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_base(&cmd.cmd.pg,
563 u64_lo32(pmu->seq_buf.gpu_va));
564 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_offset(&cmd.cmd.pg,
565 (u8)(pmu->seq_buf.gpu_va & 0xFF));
566 g->ops.pmu_ver.pg_cmd_eng_buf_load_set_dma_idx(&cmd.cmd.pg,
567 PMU_DMAIDX_VIRT);
568
569 pmu->buf_loaded = false;
570 nvgpu_pmu_dbg(g, "cmd post PMU_PG_CMD_ID_ENG_BUF_LOAD PMU_PGENG_GR_BUFFER_IDX_ZBC");
571 nvgpu_pmu_state_change(g, PMU_STATE_LOADING_ZBC, false);
572 err = nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
573 pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
574 if (err)
575 nvgpu_err(g, "CMD LOAD PMU_PGENG_GR_BUFFER_IDX_ZBC failed\n");
576}
577
578/* stats */
579int nvgpu_pmu_get_pg_stats(struct gk20a *g, u32 pg_engine_id,
580 struct pmu_pg_stats_data *pg_stat_data)
581{
582 struct nvgpu_pmu *pmu = &g->pmu;
583 u32 pg_engine_id_list = 0;
584
585 if (!pmu->initialized) {
586 pg_stat_data->ingating_time = 0;
587 pg_stat_data->ungating_time = 0;
588 pg_stat_data->gating_cnt = 0;
589 return 0;
590 }
591
592 if (g->ops.pmu.pmu_pg_supported_engines_list)
593 pg_engine_id_list = g->ops.pmu.pmu_pg_supported_engines_list(g);
594
595 if (BIT(pg_engine_id) & pg_engine_id_list)
596 g->ops.pmu.pmu_elpg_statistics(g, pg_engine_id,
597 pg_stat_data);
598
599 return 0;
600}
601
602/* AELPG */
603static void ap_callback_init_and_enable_ctrl(
604 struct gk20a *g, struct pmu_msg *msg,
605 void *param, u32 seq_desc, u32 status)
606{
607 /* Define p_ap (i.e pointer to pmu_ap structure) */
608 WARN_ON(!msg);
609
610 if (!status) {
611 switch (msg->msg.pg.ap_msg.cmn.msg_id) {
612 case PMU_AP_MSG_ID_INIT_ACK:
613 nvgpu_pmu_dbg(g, "reply PMU_AP_CMD_ID_INIT");
614 break;
615
616 default:
617 nvgpu_pmu_dbg(g,
618 "%s: Invalid Adaptive Power Message: %x\n",
619 __func__, msg->msg.pg.ap_msg.cmn.msg_id);
620 break;
621 }
622 }
623}
624
625/* Send an Adaptive Power (AP) related command to PMU */
626int nvgpu_pmu_ap_send_command(struct gk20a *g,
627 union pmu_ap_cmd *p_ap_cmd, bool b_block)
628{
629 struct nvgpu_pmu *pmu = &g->pmu;
630 /* FIXME: where is the PG structure defined?? */
631 u32 status = 0;
632 struct pmu_cmd cmd;
633 u32 seq;
634 pmu_callback p_callback = NULL;
635
636 memset(&cmd, 0, sizeof(struct pmu_cmd));
637
638 /* Copy common members */
639 cmd.hdr.unit_id = PMU_UNIT_PG;
640 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(union pmu_ap_cmd);
641
642 cmd.cmd.pg.ap_cmd.cmn.cmd_type = PMU_PG_CMD_ID_AP;
643 cmd.cmd.pg.ap_cmd.cmn.cmd_id = p_ap_cmd->cmn.cmd_id;
644
645 /* Copy other members of command */
646 switch (p_ap_cmd->cmn.cmd_id) {
647 case PMU_AP_CMD_ID_INIT:
648 nvgpu_pmu_dbg(g, "cmd post PMU_AP_CMD_ID_INIT");
649 cmd.cmd.pg.ap_cmd.init.pg_sampling_period_us =
650 p_ap_cmd->init.pg_sampling_period_us;
651 break;
652
653 case PMU_AP_CMD_ID_INIT_AND_ENABLE_CTRL:
654 nvgpu_pmu_dbg(g, "cmd post PMU_AP_CMD_ID_INIT_AND_ENABLE_CTRL");
655 cmd.cmd.pg.ap_cmd.init_and_enable_ctrl.ctrl_id =
656 p_ap_cmd->init_and_enable_ctrl.ctrl_id;
657 memcpy(
658 (void *)&(cmd.cmd.pg.ap_cmd.init_and_enable_ctrl.params),
659 (void *)&(p_ap_cmd->init_and_enable_ctrl.params),
660 sizeof(struct pmu_ap_ctrl_init_params));
661
662 p_callback = ap_callback_init_and_enable_ctrl;
663 break;
664
665 case PMU_AP_CMD_ID_ENABLE_CTRL:
666 nvgpu_pmu_dbg(g, "cmd post PMU_AP_CMD_ID_ENABLE_CTRL");
667 cmd.cmd.pg.ap_cmd.enable_ctrl.ctrl_id =
668 p_ap_cmd->enable_ctrl.ctrl_id;
669 break;
670
671 case PMU_AP_CMD_ID_DISABLE_CTRL:
672 nvgpu_pmu_dbg(g, "cmd post PMU_AP_CMD_ID_DISABLE_CTRL");
673 cmd.cmd.pg.ap_cmd.disable_ctrl.ctrl_id =
674 p_ap_cmd->disable_ctrl.ctrl_id;
675 break;
676
677 case PMU_AP_CMD_ID_KICK_CTRL:
678 nvgpu_pmu_dbg(g, "cmd post PMU_AP_CMD_ID_KICK_CTRL");
679 cmd.cmd.pg.ap_cmd.kick_ctrl.ctrl_id =
680 p_ap_cmd->kick_ctrl.ctrl_id;
681 cmd.cmd.pg.ap_cmd.kick_ctrl.skip_count =
682 p_ap_cmd->kick_ctrl.skip_count;
683 break;
684
685 default:
686 nvgpu_pmu_dbg(g, "%s: Invalid Adaptive Power command %d\n",
687 __func__, p_ap_cmd->cmn.cmd_id);
688 return 0x2f;
689 }
690
691 status = nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
692 p_callback, pmu, &seq, ~0);
693
694 if (status) {
695 nvgpu_pmu_dbg(g,
696 "%s: Unable to submit Adaptive Power Command %d\n",
697 __func__, p_ap_cmd->cmn.cmd_id);
698 goto err_return;
699 }
700
701 /* TODO: Implement blocking calls (b_block) */
702
703err_return:
704 return status;
705}
706
707int nvgpu_aelpg_init(struct gk20a *g)
708{
709 int status = 0;
710
711 /* Remove reliance on app_ctrl field. */
712 union pmu_ap_cmd ap_cmd;
713
714 /* TODO: Check for elpg being ready? */
715 ap_cmd.init.cmd_id = PMU_AP_CMD_ID_INIT;
716 ap_cmd.init.pg_sampling_period_us = g->pmu.aelpg_param[0];
717
718 status = nvgpu_pmu_ap_send_command(g, &ap_cmd, false);
719 return status;
720}
721
722int nvgpu_aelpg_init_and_enable(struct gk20a *g, u8 ctrl_id)
723{
724 int status = 0;
725 union pmu_ap_cmd ap_cmd;
726
727 /* TODO: Probably check if ELPG is ready? */
728 ap_cmd.init_and_enable_ctrl.cmd_id = PMU_AP_CMD_ID_INIT_AND_ENABLE_CTRL;
729 ap_cmd.init_and_enable_ctrl.ctrl_id = ctrl_id;
730 ap_cmd.init_and_enable_ctrl.params.min_idle_filter_us =
731 g->pmu.aelpg_param[1];
732 ap_cmd.init_and_enable_ctrl.params.min_target_saving_us =
733 g->pmu.aelpg_param[2];
734 ap_cmd.init_and_enable_ctrl.params.power_break_even_us =
735 g->pmu.aelpg_param[3];
736 ap_cmd.init_and_enable_ctrl.params.cycles_per_sample_max =
737 g->pmu.aelpg_param[4];
738
739 switch (ctrl_id) {
740 case PMU_AP_CTRL_ID_GRAPHICS:
741 break;
742 default:
743 break;
744 }
745
746 status = nvgpu_pmu_ap_send_command(g, &ap_cmd, true);
747 return status;
748}
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-12 13:43:51 -0400