summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
diff options
context:
space:
mode:
authorArto Merilainen <amerilainen@nvidia.com>2014-03-19 03:38:25 -0400
committerDan Willemsen <dwillemsen@nvidia.com>2015-03-18 15:08:53 -0400
commita9785995d5f22aaeb659285f8aeb64d8b56982e0 (patch)
treecc75f75bcf43db316a002a7a240b81f299bf6d7f /drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
parent61efaf843c22b85424036ec98015121c08f5f16c (diff)
gpu: nvgpu: Add NVIDIA GPU Driver
This patch moves the NVIDIA GPU driver to a new location. Bug 1482562 Change-Id: I24293810b9d0f1504fd9be00135e21dad656ccb6 Signed-off-by: Arto Merilainen <amerilainen@nvidia.com> Reviewed-on: http://git-master/r/383722 Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a/pmu_gk20a.c')
-rw-r--r--drivers/gpu/nvgpu/gk20a/pmu_gk20a.c3796
1 files changed, 3796 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
new file mode 100644
index 00000000..a00499a9
--- /dev/null
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
@@ -0,0 +1,3796 @@
1/*
2 * drivers/video/tegra/host/gk20a/pmu_gk20a.c
3 *
4 * GK20A PMU (aka. gPMU outside gk20a context)
5 *
6 * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * more details.
16 *
17 * You should have received a copy of the GNU General Public License along with
18 * this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20 */
21
22#include <linux/delay.h> /* for mdelay */
23#include <linux/firmware.h>
24#include <linux/clk.h>
25#include <linux/module.h>
26#include <linux/debugfs.h>
27#include <linux/dma-mapping.h>
28
29#include "gk20a.h"
30#include "hw_mc_gk20a.h"
31#include "hw_pwr_gk20a.h"
32#include "hw_top_gk20a.h"
33
34#define GK20A_PMU_UCODE_IMAGE "gpmu_ucode.bin"
35
36#define gk20a_dbg_pmu(fmt, arg...) \
37 gk20a_dbg(gpu_dbg_pmu, fmt, ##arg)
38
39static void pmu_dump_falcon_stats(struct pmu_gk20a *pmu);
40static int gk20a_pmu_get_elpg_residency_gating(struct gk20a *g,
41 u32 *ingating_time, u32 *ungating_time, u32 *gating_cnt);
42static void gk20a_init_pmu_setup_hw2_workqueue(struct work_struct *work);
43static void pmu_save_zbc(struct gk20a *g, u32 entries);
44static void ap_callback_init_and_enable_ctrl(
45 struct gk20a *g, struct pmu_msg *msg,
46 void *param, u32 seq_desc, u32 status);
47static int gk20a_pmu_ap_send_command(struct gk20a *g,
48 union pmu_ap_cmd *p_ap_cmd, bool b_block);
49
50static u32 pmu_cmdline_size_v0(struct pmu_gk20a *pmu)
51{
52 return sizeof(struct pmu_cmdline_args_v0);
53}
54
55static u32 pmu_cmdline_size_v1(struct pmu_gk20a *pmu)
56{
57 return sizeof(struct pmu_cmdline_args_v1);
58}
59
60static void set_pmu_cmdline_args_cpufreq_v1(struct pmu_gk20a *pmu, u32 freq)
61{
62 pmu->args_v1.cpu_freq_hz = freq;
63}
64
65static void set_pmu_cmdline_args_cpufreq_v0(struct pmu_gk20a *pmu, u32 freq)
66{
67 pmu->args_v0.cpu_freq_hz = freq;
68}
69
70static void *get_pmu_cmdline_args_ptr_v1(struct pmu_gk20a *pmu)
71{
72 return (void *)(&pmu->args_v1);
73}
74
75static void *get_pmu_cmdline_args_ptr_v0(struct pmu_gk20a *pmu)
76{
77 return (void *)(&pmu->args_v0);
78}
79
80static u32 get_pmu_allocation_size_v1(struct pmu_gk20a *pmu)
81{
82 return sizeof(struct pmu_allocation_v1);
83}
84
85static u32 get_pmu_allocation_size_v0(struct pmu_gk20a *pmu)
86{
87 return sizeof(struct pmu_allocation_v0);
88}
89
90static void set_pmu_allocation_ptr_v1(struct pmu_gk20a *pmu,
91 void **pmu_alloc_ptr, void *assign_ptr)
92{
93 struct pmu_allocation_v1 **pmu_a_ptr =
94 (struct pmu_allocation_v1 **)pmu_alloc_ptr;
95 *pmu_a_ptr = (struct pmu_allocation_v1 *)assign_ptr;
96}
97
98static void set_pmu_allocation_ptr_v0(struct pmu_gk20a *pmu,
99 void **pmu_alloc_ptr, void *assign_ptr)
100{
101 struct pmu_allocation_v0 **pmu_a_ptr =
102 (struct pmu_allocation_v0 **)pmu_alloc_ptr;
103 *pmu_a_ptr = (struct pmu_allocation_v0 *)assign_ptr;
104}
105
106static void pmu_allocation_set_dmem_size_v1(struct pmu_gk20a *pmu,
107 void *pmu_alloc_ptr, u16 size)
108{
109 struct pmu_allocation_v1 *pmu_a_ptr =
110 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
111 pmu_a_ptr->alloc.dmem.size = size;
112}
113
114static void pmu_allocation_set_dmem_size_v0(struct pmu_gk20a *pmu,
115 void *pmu_alloc_ptr, u16 size)
116{
117 struct pmu_allocation_v0 *pmu_a_ptr =
118 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
119 pmu_a_ptr->alloc.dmem.size = size;
120}
121
122static u16 pmu_allocation_get_dmem_size_v1(struct pmu_gk20a *pmu,
123 void *pmu_alloc_ptr)
124{
125 struct pmu_allocation_v1 *pmu_a_ptr =
126 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
127 return pmu_a_ptr->alloc.dmem.size;
128}
129
130static u16 pmu_allocation_get_dmem_size_v0(struct pmu_gk20a *pmu,
131 void *pmu_alloc_ptr)
132{
133 struct pmu_allocation_v0 *pmu_a_ptr =
134 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
135 return pmu_a_ptr->alloc.dmem.size;
136}
137
138static u32 pmu_allocation_get_dmem_offset_v1(struct pmu_gk20a *pmu,
139 void *pmu_alloc_ptr)
140{
141 struct pmu_allocation_v1 *pmu_a_ptr =
142 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
143 return pmu_a_ptr->alloc.dmem.offset;
144}
145
146static u32 pmu_allocation_get_dmem_offset_v0(struct pmu_gk20a *pmu,
147 void *pmu_alloc_ptr)
148{
149 struct pmu_allocation_v0 *pmu_a_ptr =
150 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
151 return pmu_a_ptr->alloc.dmem.offset;
152}
153
154static u32 *pmu_allocation_get_dmem_offset_addr_v1(struct pmu_gk20a *pmu,
155 void *pmu_alloc_ptr)
156{
157 struct pmu_allocation_v1 *pmu_a_ptr =
158 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
159 return &pmu_a_ptr->alloc.dmem.offset;
160}
161
162static u32 *pmu_allocation_get_dmem_offset_addr_v0(struct pmu_gk20a *pmu,
163 void *pmu_alloc_ptr)
164{
165 struct pmu_allocation_v0 *pmu_a_ptr =
166 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
167 return &pmu_a_ptr->alloc.dmem.offset;
168}
169
170static void pmu_allocation_set_dmem_offset_v1(struct pmu_gk20a *pmu,
171 void *pmu_alloc_ptr, u32 offset)
172{
173 struct pmu_allocation_v1 *pmu_a_ptr =
174 (struct pmu_allocation_v1 *)pmu_alloc_ptr;
175 pmu_a_ptr->alloc.dmem.offset = offset;
176}
177
178static void pmu_allocation_set_dmem_offset_v0(struct pmu_gk20a *pmu,
179 void *pmu_alloc_ptr, u32 offset)
180{
181 struct pmu_allocation_v0 *pmu_a_ptr =
182 (struct pmu_allocation_v0 *)pmu_alloc_ptr;
183 pmu_a_ptr->alloc.dmem.offset = offset;
184}
185
186static void *get_pmu_msg_pmu_init_msg_ptr_v1(struct pmu_init_msg *init)
187{
188 return (void *)(&(init->pmu_init_v1));
189}
190
191static u16 get_pmu_init_msg_pmu_sw_mg_off_v1(union pmu_init_msg_pmu *init_msg)
192{
193 struct pmu_init_msg_pmu_v1 *init =
194 (struct pmu_init_msg_pmu_v1 *)(&init_msg->v1);
195 return init->sw_managed_area_offset;
196}
197
198static u16 get_pmu_init_msg_pmu_sw_mg_size_v1(union pmu_init_msg_pmu *init_msg)
199{
200 struct pmu_init_msg_pmu_v1 *init =
201 (struct pmu_init_msg_pmu_v1 *)(&init_msg->v1);
202 return init->sw_managed_area_size;
203}
204
205static void *get_pmu_msg_pmu_init_msg_ptr_v0(struct pmu_init_msg *init)
206{
207 return (void *)(&(init->pmu_init_v0));
208}
209
210static u16 get_pmu_init_msg_pmu_sw_mg_off_v0(union pmu_init_msg_pmu *init_msg)
211{
212 struct pmu_init_msg_pmu_v0 *init =
213 (struct pmu_init_msg_pmu_v0 *)(&init_msg->v0);
214 return init->sw_managed_area_offset;
215}
216
217static u16 get_pmu_init_msg_pmu_sw_mg_size_v0(union pmu_init_msg_pmu *init_msg)
218{
219 struct pmu_init_msg_pmu_v0 *init =
220 (struct pmu_init_msg_pmu_v0 *)(&init_msg->v0);
221 return init->sw_managed_area_size;
222}
223
224static u32 get_pmu_perfmon_cmd_start_size_v1(void)
225{
226 return sizeof(struct pmu_perfmon_cmd_start_v1);
227}
228
229static u32 get_pmu_perfmon_cmd_start_size_v0(void)
230{
231 return sizeof(struct pmu_perfmon_cmd_start_v0);
232}
233
234static int get_perfmon_cmd_start_offsetofvar_v1(
235 enum pmu_perfmon_cmd_start_fields field)
236{
237 switch (field) {
238 case COUNTER_ALLOC:
239 return offsetof(struct pmu_perfmon_cmd_start_v1,
240 counter_alloc);
241 default:
242 return -EINVAL;
243 break;
244 }
245 return 0;
246}
247
248static int get_perfmon_cmd_start_offsetofvar_v0(
249 enum pmu_perfmon_cmd_start_fields field)
250{
251 switch (field) {
252 case COUNTER_ALLOC:
253 return offsetof(struct pmu_perfmon_cmd_start_v0,
254 counter_alloc);
255 default:
256 return -EINVAL;
257 break;
258 }
259 return 0;
260}
261
262static u32 get_pmu_perfmon_cmd_init_size_v1(void)
263{
264 return sizeof(struct pmu_perfmon_cmd_init_v1);
265}
266
267static u32 get_pmu_perfmon_cmd_init_size_v0(void)
268{
269 return sizeof(struct pmu_perfmon_cmd_init_v0);
270}
271
272static int get_perfmon_cmd_init_offsetofvar_v1(
273 enum pmu_perfmon_cmd_start_fields field)
274{
275 switch (field) {
276 case COUNTER_ALLOC:
277 return offsetof(struct pmu_perfmon_cmd_init_v1,
278 counter_alloc);
279 default:
280 return -EINVAL;
281 break;
282 }
283 return 0;
284}
285
286static int get_perfmon_cmd_init_offsetofvar_v0(
287 enum pmu_perfmon_cmd_start_fields field)
288{
289 switch (field) {
290 case COUNTER_ALLOC:
291 return offsetof(struct pmu_perfmon_cmd_init_v0,
292 counter_alloc);
293 default:
294 return -EINVAL;
295 break;
296 }
297 return 0;
298}
299
300static void perfmon_start_set_cmd_type_v1(struct pmu_perfmon_cmd *pc, u8 value)
301{
302 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
303 start->cmd_type = value;
304}
305
306static void perfmon_start_set_cmd_type_v0(struct pmu_perfmon_cmd *pc, u8 value)
307{
308 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
309 start->cmd_type = value;
310}
311
312static void perfmon_start_set_group_id_v1(struct pmu_perfmon_cmd *pc, u8 value)
313{
314 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
315 start->group_id = value;
316}
317
318static void perfmon_start_set_group_id_v0(struct pmu_perfmon_cmd *pc, u8 value)
319{
320 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
321 start->group_id = value;
322}
323
324static void perfmon_start_set_state_id_v1(struct pmu_perfmon_cmd *pc, u8 value)
325{
326 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
327 start->state_id = value;
328}
329
330static void perfmon_start_set_state_id_v0(struct pmu_perfmon_cmd *pc, u8 value)
331{
332 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
333 start->state_id = value;
334}
335
336static void perfmon_start_set_flags_v1(struct pmu_perfmon_cmd *pc, u8 value)
337{
338 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
339 start->flags = value;
340}
341
342static void perfmon_start_set_flags_v0(struct pmu_perfmon_cmd *pc, u8 value)
343{
344 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
345 start->flags = value;
346}
347
348static u8 perfmon_start_get_flags_v1(struct pmu_perfmon_cmd *pc)
349{
350 struct pmu_perfmon_cmd_start_v1 *start = &pc->start_v1;
351 return start->flags;
352}
353
354static u8 perfmon_start_get_flags_v0(struct pmu_perfmon_cmd *pc)
355{
356 struct pmu_perfmon_cmd_start_v0 *start = &pc->start_v0;
357 return start->flags;
358}
359
360static void perfmon_cmd_init_set_sample_buffer_v1(struct pmu_perfmon_cmd *pc,
361 u16 value)
362{
363 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
364 init->sample_buffer = value;
365}
366
367static void perfmon_cmd_init_set_sample_buffer_v0(struct pmu_perfmon_cmd *pc,
368 u16 value)
369{
370 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
371 init->sample_buffer = value;
372}
373
374static void perfmon_cmd_init_set_dec_cnt_v1(struct pmu_perfmon_cmd *pc,
375 u8 value)
376{
377 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
378 init->to_decrease_count = value;
379}
380
381static void perfmon_cmd_init_set_dec_cnt_v0(struct pmu_perfmon_cmd *pc,
382 u8 value)
383{
384 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
385 init->to_decrease_count = value;
386}
387
388static void perfmon_cmd_init_set_base_cnt_id_v1(struct pmu_perfmon_cmd *pc,
389 u8 value)
390{
391 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
392 init->base_counter_id = value;
393}
394
395static void perfmon_cmd_init_set_base_cnt_id_v0(struct pmu_perfmon_cmd *pc,
396 u8 value)
397{
398 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
399 init->base_counter_id = value;
400}
401
402static void perfmon_cmd_init_set_samp_period_us_v1(struct pmu_perfmon_cmd *pc,
403 u32 value)
404{
405 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
406 init->sample_period_us = value;
407}
408
409static void perfmon_cmd_init_set_samp_period_us_v0(struct pmu_perfmon_cmd *pc,
410 u32 value)
411{
412 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
413 init->sample_period_us = value;
414}
415
416static void perfmon_cmd_init_set_num_cnt_v1(struct pmu_perfmon_cmd *pc,
417 u8 value)
418{
419 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
420 init->num_counters = value;
421}
422
423static void perfmon_cmd_init_set_num_cnt_v0(struct pmu_perfmon_cmd *pc,
424 u8 value)
425{
426 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
427 init->num_counters = value;
428}
429
430static void perfmon_cmd_init_set_mov_avg_v1(struct pmu_perfmon_cmd *pc,
431 u8 value)
432{
433 struct pmu_perfmon_cmd_init_v1 *init = &pc->init_v1;
434 init->samples_in_moving_avg = value;
435}
436
437static void perfmon_cmd_init_set_mov_avg_v0(struct pmu_perfmon_cmd *pc,
438 u8 value)
439{
440 struct pmu_perfmon_cmd_init_v0 *init = &pc->init_v0;
441 init->samples_in_moving_avg = value;
442}
443
444static void get_pmu_init_msg_pmu_queue_params_v0(struct pmu_queue *queue,
445 u32 id, void *pmu_init_msg)
446{
447 struct pmu_init_msg_pmu_v0 *init =
448 (struct pmu_init_msg_pmu_v0 *)pmu_init_msg;
449 queue->index = init->queue_info[id].index;
450 queue->offset = init->queue_info[id].offset;
451 queue->size = init->queue_info[id].size;
452}
453
454static void get_pmu_init_msg_pmu_queue_params_v1(struct pmu_queue *queue,
455 u32 id, void *pmu_init_msg)
456{
457 struct pmu_init_msg_pmu_v1 *init =
458 (struct pmu_init_msg_pmu_v1 *)pmu_init_msg;
459 queue->index = init->queue_info[id].index;
460 queue->offset = init->queue_info[id].offset;
461 queue->size = init->queue_info[id].size;
462}
463
464static void *get_pmu_sequence_in_alloc_ptr_v1(struct pmu_sequence *seq)
465{
466 return (void *)(&seq->in_v1);
467}
468
469static void *get_pmu_sequence_in_alloc_ptr_v0(struct pmu_sequence *seq)
470{
471 return (void *)(&seq->in_v0);
472}
473
474static void *get_pmu_sequence_out_alloc_ptr_v1(struct pmu_sequence *seq)
475{
476 return (void *)(&seq->out_v1);
477}
478
479static void *get_pmu_sequence_out_alloc_ptr_v0(struct pmu_sequence *seq)
480{
481 return (void *)(&seq->out_v0);
482}
483
484static int gk20a_init_pmu(struct pmu_gk20a *pmu)
485{
486 struct gk20a *g = pmu->g;
487 switch (pmu->desc->app_version) {
488 case APP_VERSION_1:
489 g->ops.pmu_ver.cmd_id_zbc_table_update = 16;
490 g->ops.pmu_ver.get_pmu_cmdline_args_size =
491 pmu_cmdline_size_v1;
492 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq =
493 set_pmu_cmdline_args_cpufreq_v1;
494 g->ops.pmu_ver.get_pmu_cmdline_args_ptr =
495 get_pmu_cmdline_args_ptr_v1;
496 g->ops.pmu_ver.get_pmu_allocation_struct_size =
497 get_pmu_allocation_size_v1;
498 g->ops.pmu_ver.set_pmu_allocation_ptr =
499 set_pmu_allocation_ptr_v1;
500 g->ops.pmu_ver.pmu_allocation_set_dmem_size =
501 pmu_allocation_set_dmem_size_v1;
502 g->ops.pmu_ver.pmu_allocation_get_dmem_size =
503 pmu_allocation_get_dmem_size_v1;
504 g->ops.pmu_ver.pmu_allocation_get_dmem_offset =
505 pmu_allocation_get_dmem_offset_v1;
506 g->ops.pmu_ver.pmu_allocation_get_dmem_offset_addr =
507 pmu_allocation_get_dmem_offset_addr_v1;
508 g->ops.pmu_ver.pmu_allocation_set_dmem_offset =
509 pmu_allocation_set_dmem_offset_v1;
510 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
511 get_pmu_init_msg_pmu_queue_params_v1;
512 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
513 get_pmu_msg_pmu_init_msg_ptr_v1;
514 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
515 get_pmu_init_msg_pmu_sw_mg_off_v1;
516 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
517 get_pmu_init_msg_pmu_sw_mg_size_v1;
518 g->ops.pmu_ver.get_pmu_perfmon_cmd_start_size =
519 get_pmu_perfmon_cmd_start_size_v1;
520 g->ops.pmu_ver.get_perfmon_cmd_start_offsetofvar =
521 get_perfmon_cmd_start_offsetofvar_v1;
522 g->ops.pmu_ver.perfmon_start_set_cmd_type =
523 perfmon_start_set_cmd_type_v1;
524 g->ops.pmu_ver.perfmon_start_set_group_id =
525 perfmon_start_set_group_id_v1;
526 g->ops.pmu_ver.perfmon_start_set_state_id =
527 perfmon_start_set_state_id_v1;
528 g->ops.pmu_ver.perfmon_start_set_flags =
529 perfmon_start_set_flags_v1;
530 g->ops.pmu_ver.perfmon_start_get_flags =
531 perfmon_start_get_flags_v1;
532 g->ops.pmu_ver.get_pmu_perfmon_cmd_init_size =
533 get_pmu_perfmon_cmd_init_size_v1;
534 g->ops.pmu_ver.get_perfmon_cmd_init_offsetofvar =
535 get_perfmon_cmd_init_offsetofvar_v1;
536 g->ops.pmu_ver.perfmon_cmd_init_set_sample_buffer =
537 perfmon_cmd_init_set_sample_buffer_v1;
538 g->ops.pmu_ver.perfmon_cmd_init_set_dec_cnt =
539 perfmon_cmd_init_set_dec_cnt_v1;
540 g->ops.pmu_ver.perfmon_cmd_init_set_base_cnt_id =
541 perfmon_cmd_init_set_base_cnt_id_v1;
542 g->ops.pmu_ver.perfmon_cmd_init_set_samp_period_us =
543 perfmon_cmd_init_set_samp_period_us_v1;
544 g->ops.pmu_ver.perfmon_cmd_init_set_num_cnt =
545 perfmon_cmd_init_set_num_cnt_v1;
546 g->ops.pmu_ver.perfmon_cmd_init_set_mov_avg =
547 perfmon_cmd_init_set_mov_avg_v1;
548 g->ops.pmu_ver.get_pmu_seq_in_a_ptr =
549 get_pmu_sequence_in_alloc_ptr_v1;
550 g->ops.pmu_ver.get_pmu_seq_out_a_ptr =
551 get_pmu_sequence_out_alloc_ptr_v1;
552 break;
553 case APP_VERSION_0:
554 g->ops.pmu_ver.cmd_id_zbc_table_update = 14;
555 g->ops.pmu_ver.get_pmu_cmdline_args_size =
556 pmu_cmdline_size_v0;
557 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq =
558 set_pmu_cmdline_args_cpufreq_v0;
559 g->ops.pmu_ver.get_pmu_cmdline_args_ptr =
560 get_pmu_cmdline_args_ptr_v0;
561 g->ops.pmu_ver.get_pmu_allocation_struct_size =
562 get_pmu_allocation_size_v0;
563 g->ops.pmu_ver.set_pmu_allocation_ptr =
564 set_pmu_allocation_ptr_v0;
565 g->ops.pmu_ver.pmu_allocation_set_dmem_size =
566 pmu_allocation_set_dmem_size_v0;
567 g->ops.pmu_ver.pmu_allocation_get_dmem_size =
568 pmu_allocation_get_dmem_size_v0;
569 g->ops.pmu_ver.pmu_allocation_get_dmem_offset =
570 pmu_allocation_get_dmem_offset_v0;
571 g->ops.pmu_ver.pmu_allocation_get_dmem_offset_addr =
572 pmu_allocation_get_dmem_offset_addr_v0;
573 g->ops.pmu_ver.pmu_allocation_set_dmem_offset =
574 pmu_allocation_set_dmem_offset_v0;
575 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params =
576 get_pmu_init_msg_pmu_queue_params_v0;
577 g->ops.pmu_ver.get_pmu_msg_pmu_init_msg_ptr =
578 get_pmu_msg_pmu_init_msg_ptr_v0;
579 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_off =
580 get_pmu_init_msg_pmu_sw_mg_off_v0;
581 g->ops.pmu_ver.get_pmu_init_msg_pmu_sw_mg_size =
582 get_pmu_init_msg_pmu_sw_mg_size_v0;
583 g->ops.pmu_ver.get_pmu_perfmon_cmd_start_size =
584 get_pmu_perfmon_cmd_start_size_v0;
585 g->ops.pmu_ver.get_perfmon_cmd_start_offsetofvar =
586 get_perfmon_cmd_start_offsetofvar_v0;
587 g->ops.pmu_ver.perfmon_start_set_cmd_type =
588 perfmon_start_set_cmd_type_v0;
589 g->ops.pmu_ver.perfmon_start_set_group_id =
590 perfmon_start_set_group_id_v0;
591 g->ops.pmu_ver.perfmon_start_set_state_id =
592 perfmon_start_set_state_id_v0;
593 g->ops.pmu_ver.perfmon_start_set_flags =
594 perfmon_start_set_flags_v0;
595 g->ops.pmu_ver.perfmon_start_get_flags =
596 perfmon_start_get_flags_v0;
597 g->ops.pmu_ver.get_pmu_perfmon_cmd_init_size =
598 get_pmu_perfmon_cmd_init_size_v0;
599 g->ops.pmu_ver.get_perfmon_cmd_init_offsetofvar =
600 get_perfmon_cmd_init_offsetofvar_v0;
601 g->ops.pmu_ver.perfmon_cmd_init_set_sample_buffer =
602 perfmon_cmd_init_set_sample_buffer_v0;
603 g->ops.pmu_ver.perfmon_cmd_init_set_dec_cnt =
604 perfmon_cmd_init_set_dec_cnt_v0;
605 g->ops.pmu_ver.perfmon_cmd_init_set_base_cnt_id =
606 perfmon_cmd_init_set_base_cnt_id_v0;
607 g->ops.pmu_ver.perfmon_cmd_init_set_samp_period_us =
608 perfmon_cmd_init_set_samp_period_us_v0;
609 g->ops.pmu_ver.perfmon_cmd_init_set_num_cnt =
610 perfmon_cmd_init_set_num_cnt_v0;
611 g->ops.pmu_ver.perfmon_cmd_init_set_mov_avg =
612 perfmon_cmd_init_set_mov_avg_v0;
613 g->ops.pmu_ver.get_pmu_seq_in_a_ptr =
614 get_pmu_sequence_in_alloc_ptr_v0;
615 g->ops.pmu_ver.get_pmu_seq_out_a_ptr =
616 get_pmu_sequence_out_alloc_ptr_v0;
617 break;
618 default:
619 gk20a_err(dev_from_gk20a(pmu->g),
620 "PMU code version not supported\n");
621 return -EINVAL;
622 break;
623 }
624 return 0;
625}
626
627static void pmu_copy_from_dmem(struct pmu_gk20a *pmu,
628 u32 src, u8 *dst, u32 size, u8 port)
629{
630 struct gk20a *g = pmu->g;
631 u32 i, words, bytes;
632 u32 data, addr_mask;
633 u32 *dst_u32 = (u32*)dst;
634
635 if (size == 0) {
636 gk20a_err(dev_from_gk20a(g),
637 "size is zero");
638 return;
639 }
640
641 if (src & 0x3) {
642 gk20a_err(dev_from_gk20a(g),
643 "src (0x%08x) not 4-byte aligned", src);
644 return;
645 }
646
647 mutex_lock(&pmu->pmu_copy_lock);
648
649 words = size >> 2;
650 bytes = size & 0x3;
651
652 addr_mask = pwr_falcon_dmemc_offs_m() |
653 pwr_falcon_dmemc_blk_m();
654
655 src &= addr_mask;
656
657 gk20a_writel(g, pwr_falcon_dmemc_r(port),
658 src | pwr_falcon_dmemc_aincr_f(1));
659
660 for (i = 0; i < words; i++)
661 dst_u32[i] = gk20a_readl(g, pwr_falcon_dmemd_r(port));
662
663 if (bytes > 0) {
664 data = gk20a_readl(g, pwr_falcon_dmemd_r(port));
665 for (i = 0; i < bytes; i++) {
666 dst[(words << 2) + i] = ((u8 *)&data)[i];
667 gk20a_dbg_pmu("read: dst_u8[%d]=0x%08x",
668 i, dst[(words << 2) + i]);
669 }
670 }
671 mutex_unlock(&pmu->pmu_copy_lock);
672 return;
673}
674
675static void pmu_copy_to_dmem(struct pmu_gk20a *pmu,
676 u32 dst, u8 *src, u32 size, u8 port)
677{
678 struct gk20a *g = pmu->g;
679 u32 i, words, bytes;
680 u32 data, addr_mask;
681 u32 *src_u32 = (u32*)src;
682
683 if (size == 0) {
684 gk20a_err(dev_from_gk20a(g),
685 "size is zero");
686 return;
687 }
688
689 if (dst & 0x3) {
690 gk20a_err(dev_from_gk20a(g),
691 "dst (0x%08x) not 4-byte aligned", dst);
692 return;
693 }
694
695 mutex_lock(&pmu->pmu_copy_lock);
696
697 words = size >> 2;
698 bytes = size & 0x3;
699
700 addr_mask = pwr_falcon_dmemc_offs_m() |
701 pwr_falcon_dmemc_blk_m();
702
703 dst &= addr_mask;
704
705 gk20a_writel(g, pwr_falcon_dmemc_r(port),
706 dst | pwr_falcon_dmemc_aincw_f(1));
707
708 for (i = 0; i < words; i++)
709 gk20a_writel(g, pwr_falcon_dmemd_r(port), src_u32[i]);
710
711 if (bytes > 0) {
712 data = 0;
713 for (i = 0; i < bytes; i++)
714 ((u8 *)&data)[i] = src[(words << 2) + i];
715 gk20a_writel(g, pwr_falcon_dmemd_r(port), data);
716 }
717
718 data = gk20a_readl(g, pwr_falcon_dmemc_r(port)) & addr_mask;
719 size = ALIGN(size, 4);
720 if (data != dst + size) {
721 gk20a_err(dev_from_gk20a(g),
722 "copy failed. bytes written %d, expected %d",
723 data - dst, size);
724 }
725 mutex_unlock(&pmu->pmu_copy_lock);
726 return;
727}
728
729static int pmu_idle(struct pmu_gk20a *pmu)
730{
731 struct gk20a *g = pmu->g;
732 unsigned long end_jiffies = jiffies +
733 msecs_to_jiffies(2000);
734 u32 idle_stat;
735
736 /* wait for pmu idle */
737 do {
738 idle_stat = gk20a_readl(g, pwr_falcon_idlestate_r());
739
740 if (pwr_falcon_idlestate_falcon_busy_v(idle_stat) == 0 &&
741 pwr_falcon_idlestate_ext_busy_v(idle_stat) == 0) {
742 break;
743 }
744
745 if (time_after_eq(jiffies, end_jiffies)) {
746 gk20a_err(dev_from_gk20a(g),
747 "timeout waiting pmu idle : 0x%08x",
748 idle_stat);
749 return -EBUSY;
750 }
751 usleep_range(100, 200);
752 } while (1);
753
754 gk20a_dbg_fn("done");
755 return 0;
756}
757
758static void pmu_enable_irq(struct pmu_gk20a *pmu, bool enable)
759{
760 struct gk20a *g = pmu->g;
761
762 gk20a_dbg_fn("");
763
764 gk20a_writel(g, mc_intr_mask_0_r(),
765 gk20a_readl(g, mc_intr_mask_0_r()) &
766 ~mc_intr_mask_0_pmu_enabled_f());
767 gk20a_writel(g, mc_intr_mask_1_r(),
768 gk20a_readl(g, mc_intr_mask_1_r()) &
769 ~mc_intr_mask_1_pmu_enabled_f());
770
771 gk20a_writel(g, pwr_falcon_irqmclr_r(),
772 pwr_falcon_irqmclr_gptmr_f(1) |
773 pwr_falcon_irqmclr_wdtmr_f(1) |
774 pwr_falcon_irqmclr_mthd_f(1) |
775 pwr_falcon_irqmclr_ctxsw_f(1) |
776 pwr_falcon_irqmclr_halt_f(1) |
777 pwr_falcon_irqmclr_exterr_f(1) |
778 pwr_falcon_irqmclr_swgen0_f(1) |
779 pwr_falcon_irqmclr_swgen1_f(1) |
780 pwr_falcon_irqmclr_ext_f(0xff));
781
782 if (enable) {
783 /* dest 0=falcon, 1=host; level 0=irq0, 1=irq1 */
784 gk20a_writel(g, pwr_falcon_irqdest_r(),
785 pwr_falcon_irqdest_host_gptmr_f(0) |
786 pwr_falcon_irqdest_host_wdtmr_f(1) |
787 pwr_falcon_irqdest_host_mthd_f(0) |
788 pwr_falcon_irqdest_host_ctxsw_f(0) |
789 pwr_falcon_irqdest_host_halt_f(1) |
790 pwr_falcon_irqdest_host_exterr_f(0) |
791 pwr_falcon_irqdest_host_swgen0_f(1) |
792 pwr_falcon_irqdest_host_swgen1_f(0) |
793 pwr_falcon_irqdest_host_ext_f(0xff) |
794 pwr_falcon_irqdest_target_gptmr_f(1) |
795 pwr_falcon_irqdest_target_wdtmr_f(0) |
796 pwr_falcon_irqdest_target_mthd_f(0) |
797 pwr_falcon_irqdest_target_ctxsw_f(0) |
798 pwr_falcon_irqdest_target_halt_f(0) |
799 pwr_falcon_irqdest_target_exterr_f(0) |
800 pwr_falcon_irqdest_target_swgen0_f(0) |
801 pwr_falcon_irqdest_target_swgen1_f(0) |
802 pwr_falcon_irqdest_target_ext_f(0xff));
803
804 /* 0=disable, 1=enable */
805 gk20a_writel(g, pwr_falcon_irqmset_r(),
806 pwr_falcon_irqmset_gptmr_f(1) |
807 pwr_falcon_irqmset_wdtmr_f(1) |
808 pwr_falcon_irqmset_mthd_f(0) |
809 pwr_falcon_irqmset_ctxsw_f(0) |
810 pwr_falcon_irqmset_halt_f(1) |
811 pwr_falcon_irqmset_exterr_f(1) |
812 pwr_falcon_irqmset_swgen0_f(1) |
813 pwr_falcon_irqmset_swgen1_f(1));
814
815 gk20a_writel(g, mc_intr_mask_0_r(),
816 gk20a_readl(g, mc_intr_mask_0_r()) |
817 mc_intr_mask_0_pmu_enabled_f());
818 }
819
820 gk20a_dbg_fn("done");
821}
822
823static int pmu_enable_hw(struct pmu_gk20a *pmu, bool enable)
824{
825 struct gk20a *g = pmu->g;
826
827 gk20a_dbg_fn("");
828
829 if (enable) {
830 int retries = GR_IDLE_CHECK_MAX / GR_IDLE_CHECK_DEFAULT;
831 gk20a_enable(g, mc_enable_pwr_enabled_f());
832
833 do {
834 u32 w = gk20a_readl(g, pwr_falcon_dmactl_r()) &
835 (pwr_falcon_dmactl_dmem_scrubbing_m() |
836 pwr_falcon_dmactl_imem_scrubbing_m());
837
838 if (!w) {
839 gk20a_dbg_fn("done");
840 return 0;
841 }
842 udelay(GR_IDLE_CHECK_DEFAULT);
843 } while (--retries || !tegra_platform_is_silicon());
844
845 gk20a_disable(g, mc_enable_pwr_enabled_f());
846 gk20a_err(dev_from_gk20a(g), "Falcon mem scrubbing timeout");
847
848 return -ETIMEDOUT;
849 } else {
850 gk20a_disable(g, mc_enable_pwr_enabled_f());
851 return 0;
852 }
853}
854
855static int pmu_enable(struct pmu_gk20a *pmu, bool enable)
856{
857 struct gk20a *g = pmu->g;
858 u32 pmc_enable;
859 int err;
860
861 gk20a_dbg_fn("");
862
863 if (!enable) {
864 pmc_enable = gk20a_readl(g, mc_enable_r());
865 if (mc_enable_pwr_v(pmc_enable) !=
866 mc_enable_pwr_disabled_v()) {
867
868 pmu_enable_irq(pmu, false);
869 pmu_enable_hw(pmu, false);
870 }
871 } else {
872 err = pmu_enable_hw(pmu, true);
873 if (err)
874 return err;
875
876 /* TBD: post reset */
877
878 err = pmu_idle(pmu);
879 if (err)
880 return err;
881
882 pmu_enable_irq(pmu, true);
883 }
884
885 gk20a_dbg_fn("done");
886 return 0;
887}
888
889static int pmu_reset(struct pmu_gk20a *pmu)
890{
891 int err;
892
893 err = pmu_idle(pmu);
894 if (err)
895 return err;
896
897 /* TBD: release pmu hw mutex */
898
899 err = pmu_enable(pmu, false);
900 if (err)
901 return err;
902
903 /* TBD: cancel all sequences */
904 /* TBD: init all sequences and state tables */
905 /* TBD: restore pre-init message handler */
906
907 err = pmu_enable(pmu, true);
908 if (err)
909 return err;
910
911 return 0;
912}
913
914static int pmu_bootstrap(struct pmu_gk20a *pmu)
915{
916 struct gk20a *g = pmu->g;
917 struct gk20a_platform *platform = platform_get_drvdata(g->dev);
918 struct mm_gk20a *mm = &g->mm;
919 struct pmu_ucode_desc *desc = pmu->desc;
920 u64 addr_code, addr_data, addr_load;
921 u32 i, blocks, addr_args;
922
923 gk20a_dbg_fn("");
924
925 gk20a_writel(g, pwr_falcon_itfen_r(),
926 gk20a_readl(g, pwr_falcon_itfen_r()) |
927 pwr_falcon_itfen_ctxen_enable_f());
928 gk20a_writel(g, pwr_pmu_new_instblk_r(),
929 pwr_pmu_new_instblk_ptr_f(
930 mm->pmu.inst_block.cpu_pa >> 12) |
931 pwr_pmu_new_instblk_valid_f(1) |
932 pwr_pmu_new_instblk_target_sys_coh_f());
933
934 /* TBD: load all other surfaces */
935
936 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq(pmu,
937 clk_get_rate(platform->clk[1]));
938
939 addr_args = (pwr_falcon_hwcfg_dmem_size_v(
940 gk20a_readl(g, pwr_falcon_hwcfg_r()))
941 << GK20A_PMU_DMEM_BLKSIZE2) -
942 g->ops.pmu_ver.get_pmu_cmdline_args_size(pmu);
943
944 pmu_copy_to_dmem(pmu, addr_args,
945 (u8 *)(g->ops.pmu_ver.get_pmu_cmdline_args_ptr(pmu)),
946 g->ops.pmu_ver.get_pmu_cmdline_args_size(pmu), 0);
947
948 gk20a_writel(g, pwr_falcon_dmemc_r(0),
949 pwr_falcon_dmemc_offs_f(0) |
950 pwr_falcon_dmemc_blk_f(0) |
951 pwr_falcon_dmemc_aincw_f(1));
952
953 addr_code = u64_lo32((pmu->ucode.pmu_va +
954 desc->app_start_offset +
955 desc->app_resident_code_offset) >> 8) ;
956 addr_data = u64_lo32((pmu->ucode.pmu_va +
957 desc->app_start_offset +
958 desc->app_resident_data_offset) >> 8);
959 addr_load = u64_lo32((pmu->ucode.pmu_va +
960 desc->bootloader_start_offset) >> 8);
961
962 gk20a_writel(g, pwr_falcon_dmemd_r(0), GK20A_PMU_DMAIDX_UCODE);
963 gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code);
964 gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_size);
965 gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_code_size);
966 gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_imem_entry);
967 gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_data);
968 gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_data_size);
969 gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code);
970 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x1);
971 gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_args);
972
973 gk20a_writel(g, pwr_falcon_dmatrfbase_r(),
974 addr_load - (desc->bootloader_imem_offset >> 8));
975
976 blocks = ((desc->bootloader_size + 0xFF) & ~0xFF) >> 8;
977
978 for (i = 0; i < blocks; i++) {
979 gk20a_writel(g, pwr_falcon_dmatrfmoffs_r(),
980 desc->bootloader_imem_offset + (i << 8));
981 gk20a_writel(g, pwr_falcon_dmatrffboffs_r(),
982 desc->bootloader_imem_offset + (i << 8));
983 gk20a_writel(g, pwr_falcon_dmatrfcmd_r(),
984 pwr_falcon_dmatrfcmd_imem_f(1) |
985 pwr_falcon_dmatrfcmd_write_f(0) |
986 pwr_falcon_dmatrfcmd_size_f(6) |
987 pwr_falcon_dmatrfcmd_ctxdma_f(GK20A_PMU_DMAIDX_UCODE));
988 }
989
990 gk20a_writel(g, pwr_falcon_bootvec_r(),
991 pwr_falcon_bootvec_vec_f(desc->bootloader_entry_point));
992
993 gk20a_writel(g, pwr_falcon_cpuctl_r(),
994 pwr_falcon_cpuctl_startcpu_f(1));
995
996 gk20a_writel(g, pwr_falcon_os_r(), desc->app_version);
997
998 return 0;
999}
1000
1001static void pmu_seq_init(struct pmu_gk20a *pmu)
1002{
1003 u32 i;
1004
1005 memset(pmu->seq, 0,
1006 sizeof(struct pmu_sequence) * PMU_MAX_NUM_SEQUENCES);
1007 memset(pmu->pmu_seq_tbl, 0,
1008 sizeof(pmu->pmu_seq_tbl));
1009
1010 for (i = 0; i < PMU_MAX_NUM_SEQUENCES; i++)
1011 pmu->seq[i].id = i;
1012}
1013
1014static int pmu_seq_acquire(struct pmu_gk20a *pmu,
1015 struct pmu_sequence **pseq)
1016{
1017 struct gk20a *g = pmu->g;
1018 struct pmu_sequence *seq;
1019 u32 index;
1020
1021 mutex_lock(&pmu->pmu_seq_lock);
1022 index = find_first_zero_bit(pmu->pmu_seq_tbl,
1023 sizeof(pmu->pmu_seq_tbl));
1024 if (index >= sizeof(pmu->pmu_seq_tbl)) {
1025 gk20a_err(dev_from_gk20a(g),
1026 "no free sequence available");
1027 mutex_unlock(&pmu->pmu_seq_lock);
1028 return -EAGAIN;
1029 }
1030 set_bit(index, pmu->pmu_seq_tbl);
1031 mutex_unlock(&pmu->pmu_seq_lock);
1032
1033 seq = &pmu->seq[index];
1034 seq->state = PMU_SEQ_STATE_PENDING;
1035
1036 *pseq = seq;
1037 return 0;
1038}
1039
1040static void pmu_seq_release(struct pmu_gk20a *pmu,
1041 struct pmu_sequence *seq)
1042{
1043 struct gk20a *g = pmu->g;
1044 seq->state = PMU_SEQ_STATE_FREE;
1045 seq->desc = PMU_INVALID_SEQ_DESC;
1046 seq->callback = NULL;
1047 seq->cb_params = NULL;
1048 seq->msg = NULL;
1049 seq->out_payload = NULL;
1050 g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu,
1051 g->ops.pmu_ver.get_pmu_seq_in_a_ptr(seq), 0);
1052 g->ops.pmu_ver.pmu_allocation_set_dmem_size(pmu,
1053 g->ops.pmu_ver.get_pmu_seq_out_a_ptr(seq), 0);
1054
1055 clear_bit(seq->id, pmu->pmu_seq_tbl);
1056}
1057
1058static int pmu_queue_init(struct pmu_gk20a *pmu,
1059 u32 id, union pmu_init_msg_pmu *init)
1060{
1061 struct gk20a *g = pmu->g;
1062 struct pmu_queue *queue = &pmu->queue[id];
1063 queue->id = id;
1064 g->ops.pmu_ver.get_pmu_init_msg_pmu_queue_params(queue, id, init);
1065
1066 queue->mutex_id = id;
1067 mutex_init(&queue->mutex);
1068
1069 gk20a_dbg_pmu("queue %d: index %d, offset 0x%08x, size 0x%08x",
1070 id, queue->index, queue->offset, queue->size);
1071
1072 return 0;
1073}
1074
1075static int pmu_queue_head(struct pmu_gk20a *pmu, struct pmu_queue *queue,
1076 u32 *head, bool set)
1077{
1078 struct gk20a *g = pmu->g;
1079
1080 BUG_ON(!head);
1081
1082 if (PMU_IS_COMMAND_QUEUE(queue->id)) {
1083
1084 if (queue->index >= pwr_pmu_queue_head__size_1_v())
1085 return -EINVAL;
1086
1087 if (!set)
1088 *head = pwr_pmu_queue_head_address_v(
1089 gk20a_readl(g,
1090 pwr_pmu_queue_head_r(queue->index)));
1091 else
1092 gk20a_writel(g,
1093 pwr_pmu_queue_head_r(queue->index),
1094 pwr_pmu_queue_head_address_f(*head));
1095 } else {
1096 if (!set)
1097 *head = pwr_pmu_msgq_head_val_v(
1098 gk20a_readl(g, pwr_pmu_msgq_head_r()));
1099 else
1100 gk20a_writel(g,
1101 pwr_pmu_msgq_head_r(),
1102 pwr_pmu_msgq_head_val_f(*head));
1103 }
1104
1105 return 0;
1106}
1107
1108static int pmu_queue_tail(struct pmu_gk20a *pmu, struct pmu_queue *queue,
1109 u32 *tail, bool set)
1110{
1111 struct gk20a *g = pmu->g;
1112
1113 BUG_ON(!tail);
1114
1115 if (PMU_IS_COMMAND_QUEUE(queue->id)) {
1116
1117 if (queue->index >= pwr_pmu_queue_tail__size_1_v())
1118 return -EINVAL;
1119
1120 if (!set)
1121 *tail = pwr_pmu_queue_tail_address_v(
1122 gk20a_readl(g,
1123 pwr_pmu_queue_tail_r(queue->index)));
1124 else
1125 gk20a_writel(g,
1126 pwr_pmu_queue_tail_r(queue->index),
1127 pwr_pmu_queue_tail_address_f(*tail));
1128 } else {
1129 if (!set)
1130 *tail = pwr_pmu_msgq_tail_val_v(
1131 gk20a_readl(g, pwr_pmu_msgq_tail_r()));
1132 else
1133 gk20a_writel(g,
1134 pwr_pmu_msgq_tail_r(),
1135 pwr_pmu_msgq_tail_val_f(*tail));
1136 }
1137
1138 return 0;
1139}
1140
1141static inline void pmu_queue_read(struct pmu_gk20a *pmu,
1142 u32 offset, u8 *dst, u32 size)
1143{
1144 pmu_copy_from_dmem(pmu, offset, dst, size, 0);
1145}
1146
1147static inline void pmu_queue_write(struct pmu_gk20a *pmu,
1148 u32 offset, u8 *src, u32 size)
1149{
1150 pmu_copy_to_dmem(pmu, offset, src, size, 0);
1151}
1152
1153int pmu_mutex_acquire(struct pmu_gk20a *pmu, u32 id, u32 *token)
1154{
1155 struct gk20a *g = pmu->g;
1156 struct pmu_mutex *mutex;
1157 u32 data, owner, max_retry;
1158
1159 if (!pmu->initialized)
1160 return 0;
1161
1162 BUG_ON(!token);
1163 BUG_ON(!PMU_MUTEX_ID_IS_VALID(id));
1164 BUG_ON(id > pmu->mutex_cnt);
1165
1166 mutex = &pmu->mutex[id];
1167
1168 owner = pwr_pmu_mutex_value_v(
1169 gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
1170
1171 if (*token != PMU_INVALID_MUTEX_OWNER_ID && *token == owner) {
1172 BUG_ON(mutex->ref_cnt == 0);
1173 gk20a_dbg_pmu("already acquired by owner : 0x%08x", *token);
1174 mutex->ref_cnt++;
1175 return 0;
1176 }
1177
1178 max_retry = 40;
1179 do {
1180 data = pwr_pmu_mutex_id_value_v(
1181 gk20a_readl(g, pwr_pmu_mutex_id_r()));
1182 if (data == pwr_pmu_mutex_id_value_init_v() ||
1183 data == pwr_pmu_mutex_id_value_not_avail_v()) {
1184 gk20a_warn(dev_from_gk20a(g),
1185 "fail to generate mutex token: val 0x%08x",
1186 owner);
1187 usleep_range(20, 40);
1188 continue;
1189 }
1190
1191 owner = data;
1192 gk20a_writel(g, pwr_pmu_mutex_r(mutex->index),
1193 pwr_pmu_mutex_value_f(owner));
1194
1195 data = pwr_pmu_mutex_value_v(
1196 gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
1197
1198 if (owner == data) {
1199 mutex->ref_cnt = 1;
1200 gk20a_dbg_pmu("mutex acquired: id=%d, token=0x%x",
1201 mutex->index, *token);
1202 *token = owner;
1203 return 0;
1204 } else {
1205 gk20a_dbg_info("fail to acquire mutex idx=0x%08x",
1206 mutex->index);
1207
1208 data = gk20a_readl(g, pwr_pmu_mutex_id_release_r());
1209 data = set_field(data,
1210 pwr_pmu_mutex_id_release_value_m(),
1211 pwr_pmu_mutex_id_release_value_f(owner));
1212 gk20a_writel(g, pwr_pmu_mutex_id_release_r(), data);
1213
1214 usleep_range(20, 40);
1215 continue;
1216 }
1217 } while (max_retry-- > 0);
1218
1219 return -EBUSY;
1220}
1221
1222int pmu_mutex_release(struct pmu_gk20a *pmu, u32 id, u32 *token)
1223{
1224 struct gk20a *g = pmu->g;
1225 struct pmu_mutex *mutex;
1226 u32 owner, data;
1227
1228 if (!pmu->initialized)
1229 return 0;
1230
1231 BUG_ON(!token);
1232 BUG_ON(!PMU_MUTEX_ID_IS_VALID(id));
1233 BUG_ON(id > pmu->mutex_cnt);
1234
1235 mutex = &pmu->mutex[id];
1236
1237 owner = pwr_pmu_mutex_value_v(
1238 gk20a_readl(g, pwr_pmu_mutex_r(mutex->index)));
1239
1240 if (*token != owner) {
1241 gk20a_err(dev_from_gk20a(g),
1242 "requester 0x%08x NOT match owner 0x%08x",
1243 *token, owner);
1244 return -EINVAL;
1245 }
1246
1247 if (--mutex->ref_cnt == 0) {
1248 gk20a_writel(g, pwr_pmu_mutex_r(mutex->index),
1249 pwr_pmu_mutex_value_initial_lock_f());
1250
1251 data = gk20a_readl(g, pwr_pmu_mutex_id_release_r());
1252 data = set_field(data, pwr_pmu_mutex_id_release_value_m(),
1253 pwr_pmu_mutex_id_release_value_f(owner));
1254 gk20a_writel(g, pwr_pmu_mutex_id_release_r(), data);
1255
1256 gk20a_dbg_pmu("mutex released: id=%d, token=0x%x",
1257 mutex->index, *token);
1258 }
1259
1260 return 0;
1261}
1262
1263static int pmu_queue_lock(struct pmu_gk20a *pmu,
1264 struct pmu_queue *queue)
1265{
1266 int err;
1267
1268 if (PMU_IS_MESSAGE_QUEUE(queue->id))
1269 return 0;
1270
1271 if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
1272 mutex_lock(&queue->mutex);
1273 queue->locked = true;
1274 return 0;
1275 }
1276
1277 err = pmu_mutex_acquire(pmu, queue->mutex_id,
1278 &queue->mutex_lock);
1279 if (err == 0)
1280 queue->locked = true;
1281
1282 return err;
1283}
1284
1285static int pmu_queue_unlock(struct pmu_gk20a *pmu,
1286 struct pmu_queue *queue)
1287{
1288 int err;
1289
1290 if (PMU_IS_MESSAGE_QUEUE(queue->id))
1291 return 0;
1292
1293 if (PMU_IS_SW_COMMAND_QUEUE(queue->id)) {
1294 mutex_unlock(&queue->mutex);
1295 queue->locked = false;
1296 return 0;
1297 }
1298
1299 if (queue->locked) {
1300 err = pmu_mutex_release(pmu, queue->mutex_id,
1301 &queue->mutex_lock);
1302 if (err == 0)
1303 queue->locked = false;
1304 }
1305
1306 return 0;
1307}
1308
1309/* called by pmu_read_message, no lock */
1310static bool pmu_queue_is_empty(struct pmu_gk20a *pmu,
1311 struct pmu_queue *queue)
1312{
1313 u32 head, tail;
1314
1315 pmu_queue_head(pmu, queue, &head, QUEUE_GET);
1316 if (queue->opened && queue->oflag == OFLAG_READ)
1317 tail = queue->position;
1318 else
1319 pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
1320
1321 return head == tail;
1322}
1323
1324static bool pmu_queue_has_room(struct pmu_gk20a *pmu,
1325 struct pmu_queue *queue, u32 size, bool *need_rewind)
1326{
1327 u32 head, tail, free;
1328 bool rewind = false;
1329
1330 BUG_ON(!queue->locked);
1331
1332 size = ALIGN(size, QUEUE_ALIGNMENT);
1333
1334 pmu_queue_head(pmu, queue, &head, QUEUE_GET);
1335 pmu_queue_tail(pmu, queue, &tail, QUEUE_GET);
1336
1337 if (head >= tail) {
1338 free = queue->offset + queue->size - head;
1339 free -= PMU_CMD_HDR_SIZE;
1340
1341 if (size > free) {
1342 rewind = true;
1343 head = queue->offset;
1344 }
1345 }
1346
1347 if (head < tail)
1348 free = tail - head - 1;
1349
1350 if (need_rewind)
1351 *need_rewind = rewind;
1352
1353 return size <= free;
1354}
1355
1356static int pmu_queue_push(struct pmu_gk20a *pmu,
1357 struct pmu_queue *queue, void *data, u32 size)
1358{
1359 gk20a_dbg_fn("");
1360
1361 if (!queue->opened && queue->oflag == OFLAG_WRITE){
1362 gk20a_err(dev_from_gk20a(pmu->g),
1363 "queue not opened for write");
1364 return -EINVAL;
1365 }
1366
1367 pmu_queue_write(pmu, queue->position, data, size);
1368 queue->position += ALIGN(size, QUEUE_ALIGNMENT);
1369 return 0;
1370}
1371
1372static int pmu_queue_pop(struct pmu_gk20a *pmu,
1373 struct pmu_queue *queue, void *data, u32 size,
1374 u32 *bytes_read)
1375{
1376 u32 head, tail, used;
1377
1378 *bytes_read = 0;
1379
1380 if (!queue->opened && queue->oflag == OFLAG_READ){
1381 gk20a_err(dev_from_gk20a(pmu->g),
1382 "queue not opened for read");
1383 return -EINVAL;
1384 }
1385
1386 pmu_queue_head(pmu, queue, &head, QUEUE_GET);
1387 tail = queue->position;
1388
1389 if (head == tail)
1390 return 0;
1391
1392 if (head > tail)
1393 used = head - tail;
1394 else
1395 used = queue->offset + queue->size - tail;
1396
1397 if (size > used) {
1398 gk20a_warn(dev_from_gk20a(pmu->g),
1399 "queue size smaller than request read");
1400 size = used;
1401 }
1402
1403 pmu_queue_read(pmu, tail, data, size);
1404 queue->position += ALIGN(size, QUEUE_ALIGNMENT);
1405 *bytes_read = size;
1406 return 0;
1407}
1408
1409static void pmu_queue_rewind(struct pmu_gk20a *pmu,
1410 struct pmu_queue *queue)
1411{
1412 struct pmu_cmd cmd;
1413
1414 gk20a_dbg_fn("");
1415
1416 if (!queue->opened) {
1417 gk20a_err(dev_from_gk20a(pmu->g),
1418 "queue not opened");
1419 return;
1420 }
1421
1422 if (queue->oflag == OFLAG_WRITE) {
1423 cmd.hdr.unit_id = PMU_UNIT_REWIND;
1424 cmd.hdr.size = PMU_CMD_HDR_SIZE;
1425 pmu_queue_push(pmu, queue, &cmd, cmd.hdr.size);
1426 gk20a_dbg_pmu("queue %d rewinded", queue->id);
1427 }
1428
1429 queue->position = queue->offset;
1430 return;
1431}
1432
1433/* open for read and lock the queue */
1434static int pmu_queue_open_read(struct pmu_gk20a *pmu,
1435 struct pmu_queue *queue)
1436{
1437 int err;
1438
1439 err = pmu_queue_lock(pmu, queue);
1440 if (err)
1441 return err;
1442
1443 if (queue->opened)
1444 BUG();
1445
1446 pmu_queue_tail(pmu, queue, &queue->position, QUEUE_GET);
1447 queue->oflag = OFLAG_READ;
1448 queue->opened = true;
1449
1450 return 0;
1451}
1452
1453/* open for write and lock the queue
1454 make sure there's enough free space for the write */
1455static int pmu_queue_open_write(struct pmu_gk20a *pmu,
1456 struct pmu_queue *queue, u32 size)
1457{
1458 bool rewind = false;
1459 int err;
1460
1461 err = pmu_queue_lock(pmu, queue);
1462 if (err)
1463 return err;
1464
1465 if (queue->opened)
1466 BUG();
1467
1468 if (!pmu_queue_has_room(pmu, queue, size, &rewind)) {
1469 gk20a_err(dev_from_gk20a(pmu->g), "queue full");
1470 return -EAGAIN;
1471 }
1472
1473 pmu_queue_head(pmu, queue, &queue->position, QUEUE_GET);
1474 queue->oflag = OFLAG_WRITE;
1475 queue->opened = true;
1476
1477 if (rewind)
1478 pmu_queue_rewind(pmu, queue);
1479
1480 return 0;
1481}
1482
1483/* close and unlock the queue */
1484static int pmu_queue_close(struct pmu_gk20a *pmu,
1485 struct pmu_queue *queue, bool commit)
1486{
1487 if (!queue->opened)
1488 return 0;
1489
1490 if (commit) {
1491 if (queue->oflag == OFLAG_READ) {
1492 pmu_queue_tail(pmu, queue,
1493 &queue->position, QUEUE_SET);
1494 }
1495 else {
1496 pmu_queue_head(pmu, queue,
1497 &queue->position, QUEUE_SET);
1498 }
1499 }
1500
1501 queue->opened = false;
1502
1503 pmu_queue_unlock(pmu, queue);
1504
1505 return 0;
1506}
1507
1508static void gk20a_save_pmu_sw_state(struct pmu_gk20a *pmu,
1509 struct gk20a_pmu_save_state *save)
1510{
1511 save->seq = pmu->seq;
1512 save->next_seq_desc = pmu->next_seq_desc;
1513 save->mutex = pmu->mutex;
1514 save->mutex_cnt = pmu->mutex_cnt;
1515 save->desc = pmu->desc;
1516 save->ucode = pmu->ucode;
1517 save->elpg_enable = pmu->elpg_enable;
1518 save->pg_wq = pmu->pg_wq;
1519 save->seq_buf = pmu->seq_buf;
1520 save->pg_buf = pmu->pg_buf;
1521 save->sw_ready = pmu->sw_ready;
1522 save->pg_init = pmu->pg_init;
1523}
1524
1525static void gk20a_restore_pmu_sw_state(struct pmu_gk20a *pmu,
1526 struct gk20a_pmu_save_state *save)
1527{
1528 pmu->seq = save->seq;
1529 pmu->next_seq_desc = save->next_seq_desc;
1530 pmu->mutex = save->mutex;
1531 pmu->mutex_cnt = save->mutex_cnt;
1532 pmu->desc = save->desc;
1533 pmu->ucode = save->ucode;
1534 pmu->elpg_enable = save->elpg_enable;
1535 pmu->pg_wq = save->pg_wq;
1536 pmu->seq_buf = save->seq_buf;
1537 pmu->pg_buf = save->pg_buf;
1538 pmu->sw_ready = save->sw_ready;
1539 pmu->pg_init = save->pg_init;
1540}
1541
1542void gk20a_remove_pmu_support(struct pmu_gk20a *pmu)
1543{
1544 struct gk20a_pmu_save_state save;
1545
1546 gk20a_dbg_fn("");
1547
1548 gk20a_allocator_destroy(&pmu->dmem);
1549
1550 /* Save the stuff you don't want to lose */
1551 gk20a_save_pmu_sw_state(pmu, &save);
1552
1553 /* this function is also called by pmu_destory outside gk20a deinit that
1554 releases gk20a struct so fill up with zeros here. */
1555 memset(pmu, 0, sizeof(struct pmu_gk20a));
1556
1557 /* Restore stuff you want to keep */
1558 gk20a_restore_pmu_sw_state(pmu, &save);
1559}
1560
1561int gk20a_init_pmu_reset_enable_hw(struct gk20a *g)
1562{
1563 struct pmu_gk20a *pmu = &g->pmu;
1564
1565 gk20a_dbg_fn("");
1566
1567 pmu_enable_hw(pmu, true);
1568
1569 return 0;
1570}
1571
1572static void pmu_elpg_enable_allow(struct work_struct *work);
1573
1574int gk20a_init_pmu_setup_sw(struct gk20a *g)
1575{
1576 struct pmu_gk20a *pmu = &g->pmu;
1577 struct mm_gk20a *mm = &g->mm;
1578 struct vm_gk20a *vm = &mm->pmu.vm;
1579 struct device *d = dev_from_gk20a(g);
1580 int i, err = 0;
1581 u8 *ptr;
1582 void *ucode_ptr;
1583 struct sg_table *sgt_pmu_ucode;
1584 struct sg_table *sgt_seq_buf;
1585 DEFINE_DMA_ATTRS(attrs);
1586 dma_addr_t iova;
1587
1588 gk20a_dbg_fn("");
1589
1590 if (pmu->sw_ready) {
1591 for (i = 0; i < pmu->mutex_cnt; i++) {
1592 pmu->mutex[i].id = i;
1593 pmu->mutex[i].index = i;
1594 }
1595 pmu_seq_init(pmu);
1596
1597 gk20a_dbg_fn("skip init");
1598 goto skip_init;
1599 }
1600
1601 /* no infoRom script from vbios? */
1602
1603 /* TBD: sysmon subtask */
1604
1605 pmu->mutex_cnt = pwr_pmu_mutex__size_1_v();
1606 pmu->mutex = kzalloc(pmu->mutex_cnt *
1607 sizeof(struct pmu_mutex), GFP_KERNEL);
1608 if (!pmu->mutex) {
1609 err = -ENOMEM;
1610 goto err;
1611 }
1612
1613 for (i = 0; i < pmu->mutex_cnt; i++) {
1614 pmu->mutex[i].id = i;
1615 pmu->mutex[i].index = i;
1616 }
1617
1618 pmu->seq = kzalloc(PMU_MAX_NUM_SEQUENCES *
1619 sizeof(struct pmu_sequence), GFP_KERNEL);
1620 if (!pmu->seq) {
1621 err = -ENOMEM;
1622 goto err_free_mutex;
1623 }
1624
1625 pmu_seq_init(pmu);
1626
1627 if (!g->pmu_fw) {
1628 g->pmu_fw = gk20a_request_firmware(g, GK20A_PMU_UCODE_IMAGE);
1629 if (!g->pmu_fw) {
1630 gk20a_err(d, "failed to load pmu ucode!!");
1631 err = -ENOENT;
1632 goto err_free_seq;
1633 }
1634 }
1635
1636 gk20a_dbg_fn("firmware loaded");
1637
1638 pmu->desc = (struct pmu_ucode_desc *)g->pmu_fw->data;
1639 pmu->ucode_image = (u32 *)((u8 *)pmu->desc +
1640 pmu->desc->descriptor_size);
1641
1642
1643 INIT_DELAYED_WORK(&pmu->elpg_enable, pmu_elpg_enable_allow);
1644 INIT_WORK(&pmu->pg_init, gk20a_init_pmu_setup_hw2_workqueue);
1645
1646 gk20a_init_pmu_vm(mm);
1647
1648 dma_set_attr(DMA_ATTR_READ_ONLY, &attrs);
1649 pmu->ucode.cpuva = dma_alloc_attrs(d, GK20A_PMU_UCODE_SIZE_MAX,
1650 &iova,
1651 GFP_KERNEL,
1652 &attrs);
1653 if (!pmu->ucode.cpuva) {
1654 gk20a_err(d, "failed to allocate memory\n");
1655 err = -ENOMEM;
1656 goto err_release_fw;
1657 }
1658
1659 pmu->ucode.iova = iova;
1660 pmu->seq_buf.cpuva = dma_alloc_coherent(d, GK20A_PMU_SEQ_BUF_SIZE,
1661 &iova,
1662 GFP_KERNEL);
1663 if (!pmu->seq_buf.cpuva) {
1664 gk20a_err(d, "failed to allocate memory\n");
1665 err = -ENOMEM;
1666 goto err_free_pmu_ucode;
1667 }
1668
1669 pmu->seq_buf.iova = iova;
1670 init_waitqueue_head(&pmu->pg_wq);
1671
1672 err = gk20a_get_sgtable(d, &sgt_pmu_ucode,
1673 pmu->ucode.cpuva,
1674 pmu->ucode.iova,
1675 GK20A_PMU_UCODE_SIZE_MAX);
1676 if (err) {
1677 gk20a_err(d, "failed to allocate sg table\n");
1678 goto err_free_seq_buf;
1679 }
1680
1681 pmu->ucode.pmu_va = gk20a_gmmu_map(vm, &sgt_pmu_ucode,
1682 GK20A_PMU_UCODE_SIZE_MAX,
1683 0, /* flags */
1684 gk20a_mem_flag_read_only);
1685 if (!pmu->ucode.pmu_va) {
1686 gk20a_err(d, "failed to map pmu ucode memory!!");
1687 goto err_free_ucode_sgt;
1688 }
1689
1690 err = gk20a_get_sgtable(d, &sgt_seq_buf,
1691 pmu->seq_buf.cpuva,
1692 pmu->seq_buf.iova,
1693 GK20A_PMU_SEQ_BUF_SIZE);
1694 if (err) {
1695 gk20a_err(d, "failed to allocate sg table\n");
1696 goto err_unmap_ucode;
1697 }
1698
1699 pmu->seq_buf.pmu_va = gk20a_gmmu_map(vm, &sgt_seq_buf,
1700 GK20A_PMU_SEQ_BUF_SIZE,
1701 0, /* flags */
1702 gk20a_mem_flag_none);
1703 if (!pmu->seq_buf.pmu_va) {
1704 gk20a_err(d, "failed to map pmu ucode memory!!");
1705 goto err_free_seq_buf_sgt;
1706 }
1707
1708 ptr = (u8 *)pmu->seq_buf.cpuva;
1709 if (!ptr) {
1710 gk20a_err(d, "failed to map cpu ptr for zbc buffer");
1711 goto err_unmap_seq_buf;
1712 }
1713
1714 /* TBD: remove this if ZBC save/restore is handled by PMU
1715 * end an empty ZBC sequence for now */
1716 ptr[0] = 0x16; /* opcode EXIT */
1717 ptr[1] = 0; ptr[2] = 1; ptr[3] = 0;
1718 ptr[4] = 0; ptr[5] = 0; ptr[6] = 0; ptr[7] = 0;
1719
1720 pmu->seq_buf.size = GK20A_PMU_SEQ_BUF_SIZE;
1721
1722 ucode_ptr = pmu->ucode.cpuva;
1723
1724 for (i = 0; i < (pmu->desc->app_start_offset +
1725 pmu->desc->app_size) >> 2; i++)
1726 gk20a_mem_wr32(ucode_ptr, i, pmu->ucode_image[i]);
1727
1728 gk20a_free_sgtable(&sgt_pmu_ucode);
1729 gk20a_free_sgtable(&sgt_seq_buf);
1730
1731skip_init:
1732 mutex_init(&pmu->elpg_mutex);
1733 mutex_init(&pmu->isr_mutex);
1734 mutex_init(&pmu->pmu_copy_lock);
1735 mutex_init(&pmu->pmu_seq_lock);
1736
1737 pmu->perfmon_counter.index = 3; /* GR & CE2 */
1738 pmu->perfmon_counter.group_id = PMU_DOMAIN_GROUP_PSTATE;
1739
1740 pmu->remove_support = gk20a_remove_pmu_support;
1741 err = gk20a_init_pmu(pmu);
1742 if (err) {
1743 gk20a_err(d, "failed to set function pointers\n");
1744 return err;
1745 }
1746
1747 gk20a_dbg_fn("done");
1748 return 0;
1749
1750 err_unmap_seq_buf:
1751 gk20a_gmmu_unmap(vm, pmu->seq_buf.pmu_va,
1752 GK20A_PMU_SEQ_BUF_SIZE, gk20a_mem_flag_none);
1753 err_free_seq_buf_sgt:
1754 gk20a_free_sgtable(&sgt_seq_buf);
1755 err_unmap_ucode:
1756 gk20a_gmmu_unmap(vm, pmu->ucode.pmu_va,
1757 GK20A_PMU_UCODE_SIZE_MAX, gk20a_mem_flag_none);
1758 err_free_ucode_sgt:
1759 gk20a_free_sgtable(&sgt_pmu_ucode);
1760 err_free_seq_buf:
1761 dma_free_coherent(d, GK20A_PMU_SEQ_BUF_SIZE,
1762 pmu->seq_buf.cpuva, pmu->seq_buf.iova);
1763 pmu->seq_buf.cpuva = NULL;
1764 pmu->seq_buf.iova = 0;
1765 err_free_pmu_ucode:
1766 dma_free_attrs(d, GK20A_PMU_UCODE_SIZE_MAX,
1767 pmu->ucode.cpuva, pmu->ucode.iova, &attrs);
1768 pmu->ucode.cpuva = NULL;
1769 pmu->ucode.iova = 0;
1770 err_release_fw:
1771 release_firmware(g->pmu_fw);
1772 err_free_seq:
1773 kfree(pmu->seq);
1774 err_free_mutex:
1775 kfree(pmu->mutex);
1776 err:
1777 gk20a_dbg_fn("fail");
1778 return err;
1779}
1780
1781static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
1782 void *param, u32 handle, u32 status);
1783
1784static void pmu_handle_pg_buf_config_msg(struct gk20a *g, struct pmu_msg *msg,
1785 void *param, u32 handle, u32 status)
1786{
1787 struct pmu_gk20a *pmu = param;
1788 struct pmu_pg_msg_eng_buf_stat *eng_buf_stat = &msg->msg.pg.eng_buf_stat;
1789
1790 gk20a_dbg_fn("");
1791
1792 if (status != 0) {
1793 gk20a_err(dev_from_gk20a(g), "PGENG cmd aborted");
1794 /* TBD: disable ELPG */
1795 return;
1796 }
1797
1798 if (eng_buf_stat->status == PMU_PG_MSG_ENG_BUF_FAILED) {
1799 gk20a_err(dev_from_gk20a(g), "failed to load PGENG buffer");
1800 }
1801
1802 pmu->buf_loaded = (eng_buf_stat->status == PMU_PG_MSG_ENG_BUF_LOADED);
1803 wake_up(&pmu->pg_wq);
1804}
1805
1806int gk20a_init_pmu_setup_hw1(struct gk20a *g)
1807{
1808 struct pmu_gk20a *pmu = &g->pmu;
1809 int err;
1810
1811 gk20a_dbg_fn("");
1812
1813 pmu_reset(pmu);
1814
1815 /* setup apertures - virtual */
1816 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_UCODE),
1817 pwr_fbif_transcfg_mem_type_virtual_f());
1818 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_VIRT),
1819 pwr_fbif_transcfg_mem_type_virtual_f());
1820 /* setup apertures - physical */
1821 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_VID),
1822 pwr_fbif_transcfg_mem_type_physical_f() |
1823 pwr_fbif_transcfg_target_local_fb_f());
1824 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_COH),
1825 pwr_fbif_transcfg_mem_type_physical_f() |
1826 pwr_fbif_transcfg_target_coherent_sysmem_f());
1827 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_NCOH),
1828 pwr_fbif_transcfg_mem_type_physical_f() |
1829 pwr_fbif_transcfg_target_noncoherent_sysmem_f());
1830
1831 /* TBD: load pmu ucode */
1832 err = pmu_bootstrap(pmu);
1833 if (err)
1834 return err;
1835
1836 return 0;
1837
1838}
1839
1840static int gk20a_aelpg_init(struct gk20a *g);
1841static int gk20a_aelpg_init_and_enable(struct gk20a *g, u8 ctrl_id);
1842
1843
1844static void gk20a_init_pmu_setup_hw2_workqueue(struct work_struct *work)
1845{
1846 struct pmu_gk20a *pmu = container_of(work, struct pmu_gk20a, pg_init);
1847 struct gk20a *g = pmu->g;
1848 gk20a_init_pmu_setup_hw2(g);
1849}
1850
1851int gk20a_init_pmu_setup_hw2(struct gk20a *g)
1852{
1853 struct pmu_gk20a *pmu = &g->pmu;
1854 struct mm_gk20a *mm = &g->mm;
1855 struct vm_gk20a *vm = &mm->pmu.vm;
1856 struct device *d = dev_from_gk20a(g);
1857 struct pmu_cmd cmd;
1858 u32 desc;
1859 long remain;
1860 int err;
1861 bool status;
1862 u32 size;
1863 struct sg_table *sgt_pg_buf;
1864 dma_addr_t iova;
1865
1866 gk20a_dbg_fn("");
1867
1868 if (!support_gk20a_pmu())
1869 return 0;
1870
1871 size = 0;
1872 err = gr_gk20a_fecs_get_reglist_img_size(g, &size);
1873 if (err) {
1874 gk20a_err(dev_from_gk20a(g),
1875 "fail to query fecs pg buffer size");
1876 return err;
1877 }
1878
1879 if (!pmu->sw_ready) {
1880 pmu->pg_buf.cpuva = dma_alloc_coherent(d, size,
1881 &iova,
1882 GFP_KERNEL);
1883 if (!pmu->pg_buf.cpuva) {
1884 gk20a_err(d, "failed to allocate memory\n");
1885 err = -ENOMEM;
1886 goto err;
1887 }
1888
1889 pmu->pg_buf.iova = iova;
1890 pmu->pg_buf.size = size;
1891
1892 err = gk20a_get_sgtable(d, &sgt_pg_buf,
1893 pmu->pg_buf.cpuva,
1894 pmu->pg_buf.iova,
1895 size);
1896 if (err) {
1897 gk20a_err(d, "failed to create sg table\n");
1898 goto err_free_pg_buf;
1899 }
1900
1901 pmu->pg_buf.pmu_va = gk20a_gmmu_map(vm,
1902 &sgt_pg_buf,
1903 size,
1904 0, /* flags */
1905 gk20a_mem_flag_none);
1906 if (!pmu->pg_buf.pmu_va) {
1907 gk20a_err(d, "failed to map fecs pg buffer");
1908 err = -ENOMEM;
1909 goto err_free_sgtable;
1910 }
1911
1912 gk20a_free_sgtable(&sgt_pg_buf);
1913 }
1914
1915 /*
1916 * This is the actual point at which sw setup is complete, so set the
1917 * sw_ready flag here.
1918 */
1919 pmu->sw_ready = true;
1920
1921 /* TBD: acquire pmu hw mutex */
1922
1923 /* TBD: post reset again? */
1924
1925 /* PMU_INIT message handler will send PG_INIT */
1926 remain = wait_event_timeout(
1927 pmu->pg_wq,
1928 (status = (pmu->elpg_ready &&
1929 pmu->stat_dmem_offset != 0 &&
1930 pmu->elpg_stat == PMU_ELPG_STAT_OFF)),
1931 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1932 if (status == 0) {
1933 gk20a_err(dev_from_gk20a(g),
1934 "PG_INIT_ACK failed, remaining timeout : 0x%lx", remain);
1935 pmu_dump_falcon_stats(pmu);
1936 return -EBUSY;
1937 }
1938
1939 err = gr_gk20a_fecs_set_reglist_bind_inst(g, mm->pmu.inst_block.cpu_pa);
1940 if (err) {
1941 gk20a_err(dev_from_gk20a(g),
1942 "fail to bind pmu inst to gr");
1943 return err;
1944 }
1945
1946 err = gr_gk20a_fecs_set_reglist_virual_addr(g, pmu->pg_buf.pmu_va);
1947 if (err) {
1948 gk20a_err(dev_from_gk20a(g),
1949 "fail to set pg buffer pmu va");
1950 return err;
1951 }
1952
1953 memset(&cmd, 0, sizeof(struct pmu_cmd));
1954 cmd.hdr.unit_id = PMU_UNIT_PG;
1955 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_eng_buf_load);
1956 cmd.cmd.pg.eng_buf_load.cmd_type = PMU_PG_CMD_ID_ENG_BUF_LOAD;
1957 cmd.cmd.pg.eng_buf_load.engine_id = ENGINE_GR_GK20A;
1958 cmd.cmd.pg.eng_buf_load.buf_idx = PMU_PGENG_GR_BUFFER_IDX_FECS;
1959 cmd.cmd.pg.eng_buf_load.buf_size = pmu->pg_buf.size;
1960 cmd.cmd.pg.eng_buf_load.dma_base = u64_lo32(pmu->pg_buf.pmu_va >> 8);
1961 cmd.cmd.pg.eng_buf_load.dma_offset = (u8)(pmu->pg_buf.pmu_va & 0xFF);
1962 cmd.cmd.pg.eng_buf_load.dma_idx = PMU_DMAIDX_VIRT;
1963
1964 pmu->buf_loaded = false;
1965 gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1966 pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
1967
1968 remain = wait_event_timeout(
1969 pmu->pg_wq,
1970 pmu->buf_loaded,
1971 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1972 if (!pmu->buf_loaded) {
1973 gk20a_err(dev_from_gk20a(g),
1974 "PGENG FECS buffer load failed, remaining timeout : 0x%lx",
1975 remain);
1976 return -EBUSY;
1977 }
1978
1979 memset(&cmd, 0, sizeof(struct pmu_cmd));
1980 cmd.hdr.unit_id = PMU_UNIT_PG;
1981 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_eng_buf_load);
1982 cmd.cmd.pg.eng_buf_load.cmd_type = PMU_PG_CMD_ID_ENG_BUF_LOAD;
1983 cmd.cmd.pg.eng_buf_load.engine_id = ENGINE_GR_GK20A;
1984 cmd.cmd.pg.eng_buf_load.buf_idx = PMU_PGENG_GR_BUFFER_IDX_ZBC;
1985 cmd.cmd.pg.eng_buf_load.buf_size = pmu->seq_buf.size;
1986 cmd.cmd.pg.eng_buf_load.dma_base = u64_lo32(pmu->seq_buf.pmu_va >> 8);
1987 cmd.cmd.pg.eng_buf_load.dma_offset = (u8)(pmu->seq_buf.pmu_va & 0xFF);
1988 cmd.cmd.pg.eng_buf_load.dma_idx = PMU_DMAIDX_VIRT;
1989
1990 pmu->buf_loaded = false;
1991 gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
1992 pmu_handle_pg_buf_config_msg, pmu, &desc, ~0);
1993
1994 remain = wait_event_timeout(
1995 pmu->pg_wq,
1996 pmu->buf_loaded,
1997 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1998 if (!pmu->buf_loaded) {
1999 gk20a_err(dev_from_gk20a(g),
2000 "PGENG ZBC buffer load failed, remaining timeout 0x%lx",
2001 remain);
2002 return -EBUSY;
2003 }
2004
2005 /*
2006 * FIXME: To enable ELPG, we increase the PMU ext2priv timeout unit to
2007 * 7. This prevents PMU stalling on Host register accesses. Once the
2008 * cause for this hang is discovered and fixed, this WAR should be
2009 * removed.
2010 */
2011 gk20a_writel(g, 0x10a164, 0x109ff);
2012
2013 pmu->initialized = true;
2014 pmu->zbc_ready = true;
2015
2016 /* Save zbc table after PMU is initialized. */
2017 pmu_save_zbc(g, 0xf);
2018
2019 /*
2020 * We can't guarantee that gr code to enable ELPG will be
2021 * invoked, so we explicitly call disable-enable here
2022 * to enable elpg.
2023 */
2024 gk20a_pmu_disable_elpg(g);
2025
2026 if (g->elpg_enabled)
2027 gk20a_pmu_enable_elpg(g);
2028
2029 udelay(50);
2030
2031 /* Enable AELPG */
2032 if (g->aelpg_enabled) {
2033 gk20a_aelpg_init(g);
2034 gk20a_aelpg_init_and_enable(g, PMU_AP_CTRL_ID_GRAPHICS);
2035 }
2036
2037 return 0;
2038
2039 err_free_sgtable:
2040 gk20a_free_sgtable(&sgt_pg_buf);
2041 err_free_pg_buf:
2042 dma_free_coherent(d, size,
2043 pmu->pg_buf.cpuva, pmu->pg_buf.iova);
2044 pmu->pg_buf.cpuva = NULL;
2045 pmu->pg_buf.iova = 0;
2046 err:
2047 return err;
2048}
2049
2050int gk20a_init_pmu_support(struct gk20a *g)
2051{
2052 struct pmu_gk20a *pmu = &g->pmu;
2053 u32 err;
2054
2055 gk20a_dbg_fn("");
2056
2057 if (pmu->initialized)
2058 return 0;
2059
2060 pmu->g = g;
2061
2062 err = gk20a_init_pmu_reset_enable_hw(g);
2063 if (err)
2064 return err;
2065
2066 if (support_gk20a_pmu()) {
2067 err = gk20a_init_pmu_setup_sw(g);
2068 if (err)
2069 return err;
2070
2071 err = gk20a_init_pmu_setup_hw1(g);
2072 if (err)
2073 return err;
2074 }
2075
2076 return err;
2077}
2078
2079static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
2080 void *param, u32 handle, u32 status)
2081{
2082 struct pmu_gk20a *pmu = param;
2083 struct pmu_pg_msg_elpg_msg *elpg_msg = &msg->msg.pg.elpg_msg;
2084
2085 gk20a_dbg_fn("");
2086
2087 if (status != 0) {
2088 gk20a_err(dev_from_gk20a(g), "ELPG cmd aborted");
2089 /* TBD: disable ELPG */
2090 return;
2091 }
2092
2093 switch (elpg_msg->msg) {
2094 case PMU_PG_ELPG_MSG_INIT_ACK:
2095 gk20a_dbg_pmu("INIT_PG is acknowledged from PMU");
2096 pmu->elpg_ready = true;
2097 wake_up(&pmu->pg_wq);
2098 break;
2099 case PMU_PG_ELPG_MSG_ALLOW_ACK:
2100 gk20a_dbg_pmu("ALLOW is acknowledged from PMU");
2101 pmu->elpg_stat = PMU_ELPG_STAT_ON;
2102 wake_up(&pmu->pg_wq);
2103 break;
2104 case PMU_PG_ELPG_MSG_DISALLOW_ACK:
2105 gk20a_dbg_pmu("DISALLOW is acknowledged from PMU");
2106 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
2107 wake_up(&pmu->pg_wq);
2108 break;
2109 default:
2110 gk20a_err(dev_from_gk20a(g),
2111 "unsupported ELPG message : 0x%04x", elpg_msg->msg);
2112 }
2113
2114 return;
2115}
2116
2117static void pmu_handle_pg_stat_msg(struct gk20a *g, struct pmu_msg *msg,
2118 void *param, u32 handle, u32 status)
2119{
2120 struct pmu_gk20a *pmu = param;
2121
2122 gk20a_dbg_fn("");
2123
2124 if (status != 0) {
2125 gk20a_err(dev_from_gk20a(g), "ELPG cmd aborted");
2126 /* TBD: disable ELPG */
2127 return;
2128 }
2129
2130 switch (msg->msg.pg.stat.sub_msg_id) {
2131 case PMU_PG_STAT_MSG_RESP_DMEM_OFFSET:
2132 gk20a_dbg_pmu("ALLOC_DMEM_OFFSET is acknowledged from PMU");
2133 pmu->stat_dmem_offset = msg->msg.pg.stat.data;
2134 wake_up(&pmu->pg_wq);
2135 break;
2136 default:
2137 break;
2138 }
2139}
2140
2141static int pmu_init_powergating(struct pmu_gk20a *pmu)
2142{
2143 struct gk20a *g = pmu->g;
2144 struct pmu_cmd cmd;
2145 u32 seq;
2146
2147 gk20a_dbg_fn("");
2148
2149 if (tegra_cpu_is_asim()) {
2150 /* TBD: calculate threshold for silicon */
2151 gk20a_writel(g, pwr_pmu_pg_idlefilth_r(ENGINE_GR_GK20A),
2152 PMU_PG_IDLE_THRESHOLD_SIM);
2153 gk20a_writel(g, pwr_pmu_pg_ppuidlefilth_r(ENGINE_GR_GK20A),
2154 PMU_PG_POST_POWERUP_IDLE_THRESHOLD_SIM);
2155 } else {
2156 /* TBD: calculate threshold for silicon */
2157 gk20a_writel(g, pwr_pmu_pg_idlefilth_r(ENGINE_GR_GK20A),
2158 PMU_PG_IDLE_THRESHOLD);
2159 gk20a_writel(g, pwr_pmu_pg_ppuidlefilth_r(ENGINE_GR_GK20A),
2160 PMU_PG_POST_POWERUP_IDLE_THRESHOLD);
2161 }
2162
2163 /* init ELPG */
2164 memset(&cmd, 0, sizeof(struct pmu_cmd));
2165 cmd.hdr.unit_id = PMU_UNIT_PG;
2166 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
2167 cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
2168 cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
2169 cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_INIT;
2170
2171 gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
2172 pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
2173
2174 /* alloc dmem for powergating state log */
2175 pmu->stat_dmem_offset = 0;
2176 memset(&cmd, 0, sizeof(struct pmu_cmd));
2177 cmd.hdr.unit_id = PMU_UNIT_PG;
2178 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_stat);
2179 cmd.cmd.pg.stat.cmd_type = PMU_PG_CMD_ID_PG_STAT;
2180 cmd.cmd.pg.stat.engine_id = ENGINE_GR_GK20A;
2181 cmd.cmd.pg.stat.sub_cmd_id = PMU_PG_STAT_CMD_ALLOC_DMEM;
2182 cmd.cmd.pg.stat.data = 0;
2183
2184 gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
2185 pmu_handle_pg_stat_msg, pmu, &seq, ~0);
2186
2187 /* disallow ELPG initially
2188 PMU ucode requires a disallow cmd before allow cmd */
2189 pmu->elpg_stat = PMU_ELPG_STAT_ON; /* set for wait_event PMU_ELPG_STAT_OFF */
2190 memset(&cmd, 0, sizeof(struct pmu_cmd));
2191 cmd.hdr.unit_id = PMU_UNIT_PG;
2192 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
2193 cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
2194 cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
2195 cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
2196
2197 gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
2198 pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
2199
2200 /* start with elpg disabled until first enable call */
2201 pmu->elpg_refcnt = 1;
2202
2203 return 0;
2204}
2205
2206static int pmu_init_perfmon(struct pmu_gk20a *pmu)
2207{
2208 struct gk20a *g = pmu->g;
2209 struct pmu_v *pv = &g->ops.pmu_ver;
2210 struct pmu_cmd cmd;
2211 struct pmu_payload payload;
2212 u32 seq;
2213 u32 data;
2214 int err;
2215
2216 gk20a_dbg_fn("");
2217
2218 pmu->perfmon_ready = 0;
2219
2220 /* use counter #3 for GR && CE2 busy cycles */
2221 gk20a_writel(g, pwr_pmu_idle_mask_r(3),
2222 pwr_pmu_idle_mask_gr_enabled_f() |
2223 pwr_pmu_idle_mask_ce_2_enabled_f());
2224
2225 /* disable idle filtering for counters 3 and 6 */
2226 data = gk20a_readl(g, pwr_pmu_idle_ctrl_r(3));
2227 data = set_field(data, pwr_pmu_idle_ctrl_value_m() |
2228 pwr_pmu_idle_ctrl_filter_m(),
2229 pwr_pmu_idle_ctrl_value_busy_f() |
2230 pwr_pmu_idle_ctrl_filter_disabled_f());
2231 gk20a_writel(g, pwr_pmu_idle_ctrl_r(3), data);
2232
2233 /* use counter #6 for total cycles */
2234 data = gk20a_readl(g, pwr_pmu_idle_ctrl_r(6));
2235 data = set_field(data, pwr_pmu_idle_ctrl_value_m() |
2236 pwr_pmu_idle_ctrl_filter_m(),
2237 pwr_pmu_idle_ctrl_value_always_f() |
2238 pwr_pmu_idle_ctrl_filter_disabled_f());
2239 gk20a_writel(g, pwr_pmu_idle_ctrl_r(6), data);
2240
2241 /*
2242 * We don't want to disturb counters #3 and #6, which are used by
2243 * perfmon, so we add wiring also to counters #1 and #2 for
2244 * exposing raw counter readings.
2245 */
2246 gk20a_writel(g, pwr_pmu_idle_mask_r(1),
2247 pwr_pmu_idle_mask_gr_enabled_f() |
2248 pwr_pmu_idle_mask_ce_2_enabled_f());
2249
2250 data = gk20a_readl(g, pwr_pmu_idle_ctrl_r(1));
2251 data = set_field(data, pwr_pmu_idle_ctrl_value_m() |
2252 pwr_pmu_idle_ctrl_filter_m(),
2253 pwr_pmu_idle_ctrl_value_busy_f() |
2254 pwr_pmu_idle_ctrl_filter_disabled_f());
2255 gk20a_writel(g, pwr_pmu_idle_ctrl_r(1), data);
2256
2257 data = gk20a_readl(g, pwr_pmu_idle_ctrl_r(2));
2258 data = set_field(data, pwr_pmu_idle_ctrl_value_m() |
2259 pwr_pmu_idle_ctrl_filter_m(),
2260 pwr_pmu_idle_ctrl_value_always_f() |
2261 pwr_pmu_idle_ctrl_filter_disabled_f());
2262 gk20a_writel(g, pwr_pmu_idle_ctrl_r(2), data);
2263
2264 pmu->sample_buffer = 0;
2265 err = pmu->dmem.alloc(&pmu->dmem, &pmu->sample_buffer, 2 * sizeof(u16));
2266 if (err) {
2267 gk20a_err(dev_from_gk20a(g),
2268 "failed to allocate perfmon sample buffer");
2269 return -ENOMEM;
2270 }
2271
2272 /* init PERFMON */
2273 memset(&cmd, 0, sizeof(struct pmu_cmd));
2274 cmd.hdr.unit_id = PMU_UNIT_PERFMON;
2275 cmd.hdr.size = PMU_CMD_HDR_SIZE + pv->get_pmu_perfmon_cmd_init_size();
2276 cmd.cmd.perfmon.cmd_type = PMU_PERFMON_CMD_ID_INIT;
2277 /* buffer to save counter values for pmu perfmon */
2278 pv->perfmon_cmd_init_set_sample_buffer(&cmd.cmd.perfmon,
2279 (u16)pmu->sample_buffer);
2280 /* number of sample periods below lower threshold
2281 before pmu triggers perfmon decrease event
2282 TBD: = 15 */
2283 pv->perfmon_cmd_init_set_dec_cnt(&cmd.cmd.perfmon, 15);
2284 /* index of base counter, aka. always ticking counter */
2285 pv->perfmon_cmd_init_set_base_cnt_id(&cmd.cmd.perfmon, 6);
2286 /* microseconds interval between pmu polls perf counters */
2287 pv->perfmon_cmd_init_set_samp_period_us(&cmd.cmd.perfmon, 16700);
2288 /* number of perfmon counters
2289 counter #3 (GR and CE2) for gk20a */
2290 pv->perfmon_cmd_init_set_num_cnt(&cmd.cmd.perfmon, 1);
2291 /* moving average window for sample periods
2292 TBD: = 3000000 / sample_period_us = 17 */
2293 pv->perfmon_cmd_init_set_mov_avg(&cmd.cmd.perfmon, 17);
2294
2295 memset(&payload, 0, sizeof(struct pmu_payload));
2296 payload.in.buf = &pmu->perfmon_counter;
2297 payload.in.size = sizeof(struct pmu_perfmon_counter);
2298 payload.in.offset = pv->get_perfmon_cmd_init_offsetofvar(COUNTER_ALLOC);
2299
2300 gk20a_pmu_cmd_post(g, &cmd, NULL, &payload, PMU_COMMAND_QUEUE_LPQ,
2301 NULL, NULL, &seq, ~0);
2302
2303 return 0;
2304}
2305
2306static int pmu_process_init_msg(struct pmu_gk20a *pmu,
2307 struct pmu_msg *msg)
2308{
2309 struct gk20a *g = pmu->g;
2310 struct pmu_v *pv = &g->ops.pmu_ver;
2311 union pmu_init_msg_pmu *init;
2312 struct pmu_sha1_gid_data gid_data;
2313 u32 i, tail = 0;
2314
2315 tail = pwr_pmu_msgq_tail_val_v(
2316 gk20a_readl(g, pwr_pmu_msgq_tail_r()));
2317
2318 pmu_copy_from_dmem(pmu, tail,
2319 (u8 *)&msg->hdr, PMU_MSG_HDR_SIZE, 0);
2320
2321 if (msg->hdr.unit_id != PMU_UNIT_INIT) {
2322 gk20a_err(dev_from_gk20a(g),
2323 "expecting init msg");
2324 return -EINVAL;
2325 }
2326
2327 pmu_copy_from_dmem(pmu, tail + PMU_MSG_HDR_SIZE,
2328 (u8 *)&msg->msg, msg->hdr.size - PMU_MSG_HDR_SIZE, 0);
2329
2330 if (msg->msg.init.msg_type != PMU_INIT_MSG_TYPE_PMU_INIT) {
2331 gk20a_err(dev_from_gk20a(g),
2332 "expecting init msg");
2333 return -EINVAL;
2334 }
2335
2336 tail += ALIGN(msg->hdr.size, PMU_DMEM_ALIGNMENT);
2337 gk20a_writel(g, pwr_pmu_msgq_tail_r(),
2338 pwr_pmu_msgq_tail_val_f(tail));
2339
2340 init = pv->get_pmu_msg_pmu_init_msg_ptr(&(msg->msg.init));
2341 if (!pmu->gid_info.valid) {
2342
2343 pmu_copy_from_dmem(pmu,
2344 pv->get_pmu_init_msg_pmu_sw_mg_off(init),
2345 (u8 *)&gid_data,
2346 sizeof(struct pmu_sha1_gid_data), 0);
2347
2348 pmu->gid_info.valid =
2349 (*(u32 *)gid_data.signature == PMU_SHA1_GID_SIGNATURE);
2350
2351 if (pmu->gid_info.valid) {
2352
2353 BUG_ON(sizeof(pmu->gid_info.gid) !=
2354 sizeof(gid_data.gid));
2355
2356 memcpy(pmu->gid_info.gid, gid_data.gid,
2357 sizeof(pmu->gid_info.gid));
2358 }
2359 }
2360
2361 for (i = 0; i < PMU_QUEUE_COUNT; i++)
2362 pmu_queue_init(pmu, i, init);
2363
2364 gk20a_allocator_init(&pmu->dmem, "gk20a_pmu_dmem",
2365 pv->get_pmu_init_msg_pmu_sw_mg_off(init),
2366 pv->get_pmu_init_msg_pmu_sw_mg_size(init),
2367 PMU_DMEM_ALLOC_ALIGNMENT);
2368
2369 pmu->pmu_ready = true;
2370
2371 return 0;
2372}
2373
2374static bool pmu_read_message(struct pmu_gk20a *pmu, struct pmu_queue *queue,
2375 struct pmu_msg *msg, int *status)
2376{
2377 struct gk20a *g = pmu->g;
2378 u32 read_size, bytes_read;
2379 int err;
2380
2381 *status = 0;
2382
2383 if (pmu_queue_is_empty(pmu, queue))
2384 return false;
2385
2386 err = pmu_queue_open_read(pmu, queue);
2387 if (err) {
2388 gk20a_err(dev_from_gk20a(g),
2389 "fail to open queue %d for read", queue->id);
2390 *status = err;
2391 return false;
2392 }
2393
2394 err = pmu_queue_pop(pmu, queue, &msg->hdr,
2395 PMU_MSG_HDR_SIZE, &bytes_read);
2396 if (err || bytes_read != PMU_MSG_HDR_SIZE) {
2397 gk20a_err(dev_from_gk20a(g),
2398 "fail to read msg from queue %d", queue->id);
2399 *status = err | -EINVAL;
2400 goto clean_up;
2401 }
2402
2403 if (msg->hdr.unit_id == PMU_UNIT_REWIND) {
2404 pmu_queue_rewind(pmu, queue);
2405 /* read again after rewind */
2406 err = pmu_queue_pop(pmu, queue, &msg->hdr,
2407 PMU_MSG_HDR_SIZE, &bytes_read);
2408 if (err || bytes_read != PMU_MSG_HDR_SIZE) {
2409 gk20a_err(dev_from_gk20a(g),
2410 "fail to read msg from queue %d", queue->id);
2411 *status = err | -EINVAL;
2412 goto clean_up;
2413 }
2414 }
2415
2416 if (!PMU_UNIT_ID_IS_VALID(msg->hdr.unit_id)) {
2417 gk20a_err(dev_from_gk20a(g),
2418 "read invalid unit_id %d from queue %d",
2419 msg->hdr.unit_id, queue->id);
2420 *status = -EINVAL;
2421 goto clean_up;
2422 }
2423
2424 if (msg->hdr.size > PMU_MSG_HDR_SIZE) {
2425 read_size = msg->hdr.size - PMU_MSG_HDR_SIZE;
2426 err = pmu_queue_pop(pmu, queue, &msg->msg,
2427 read_size, &bytes_read);
2428 if (err || bytes_read != read_size) {
2429 gk20a_err(dev_from_gk20a(g),
2430 "fail to read msg from queue %d", queue->id);
2431 *status = err;
2432 goto clean_up;
2433 }
2434 }
2435
2436 err = pmu_queue_close(pmu, queue, true);
2437 if (err) {
2438 gk20a_err(dev_from_gk20a(g),
2439 "fail to close queue %d", queue->id);
2440 *status = err;
2441 return false;
2442 }
2443
2444 return true;
2445
2446clean_up:
2447 err = pmu_queue_close(pmu, queue, false);
2448 if (err)
2449 gk20a_err(dev_from_gk20a(g),
2450 "fail to close queue %d", queue->id);
2451 return false;
2452}
2453
2454static int pmu_response_handle(struct pmu_gk20a *pmu,
2455 struct pmu_msg *msg)
2456{
2457 struct gk20a *g = pmu->g;
2458 struct pmu_sequence *seq;
2459 struct pmu_v *pv = &g->ops.pmu_ver;
2460 int ret = 0;
2461
2462 gk20a_dbg_fn("");
2463
2464 seq = &pmu->seq[msg->hdr.seq_id];
2465 if (seq->state != PMU_SEQ_STATE_USED &&
2466 seq->state != PMU_SEQ_STATE_CANCELLED) {
2467 gk20a_err(dev_from_gk20a(g),
2468 "msg for an unknown sequence %d", seq->id);
2469 return -EINVAL;
2470 }
2471
2472 if (msg->hdr.unit_id == PMU_UNIT_RC &&
2473 msg->msg.rc.msg_type == PMU_RC_MSG_TYPE_UNHANDLED_CMD) {
2474 gk20a_err(dev_from_gk20a(g),
2475 "unhandled cmd: seq %d", seq->id);
2476 }
2477 else if (seq->state != PMU_SEQ_STATE_CANCELLED) {
2478 if (seq->msg) {
2479 if (seq->msg->hdr.size >= msg->hdr.size) {
2480 memcpy(seq->msg, msg, msg->hdr.size);
2481 if (pv->pmu_allocation_get_dmem_size(pmu,
2482 pv->get_pmu_seq_out_a_ptr(seq)) != 0) {
2483 pmu_copy_from_dmem(pmu,
2484 pv->pmu_allocation_get_dmem_offset(pmu,
2485 pv->get_pmu_seq_out_a_ptr(seq)),
2486 seq->out_payload,
2487 pv->pmu_allocation_get_dmem_size(pmu,
2488 pv->get_pmu_seq_out_a_ptr(seq)), 0);
2489 }
2490 } else {
2491 gk20a_err(dev_from_gk20a(g),
2492 "sequence %d msg buffer too small",
2493 seq->id);
2494 }
2495 }
2496 } else
2497 seq->callback = NULL;
2498 if (pv->pmu_allocation_get_dmem_size(pmu,
2499 pv->get_pmu_seq_in_a_ptr(seq)) != 0)
2500 pmu->dmem.free(&pmu->dmem,
2501 pv->pmu_allocation_get_dmem_offset(pmu,
2502 pv->get_pmu_seq_in_a_ptr(seq)),
2503 pv->pmu_allocation_get_dmem_size(pmu,
2504 pv->get_pmu_seq_in_a_ptr(seq)));
2505 if (pv->pmu_allocation_get_dmem_size(pmu,
2506 pv->get_pmu_seq_out_a_ptr(seq)) != 0)
2507 pmu->dmem.free(&pmu->dmem,
2508 pv->pmu_allocation_get_dmem_offset(pmu,
2509 pv->get_pmu_seq_out_a_ptr(seq)),
2510 pv->pmu_allocation_get_dmem_size(pmu,
2511 pv->get_pmu_seq_out_a_ptr(seq)));
2512
2513 if (seq->callback)
2514 seq->callback(g, msg, seq->cb_params, seq->desc, ret);
2515
2516 pmu_seq_release(pmu, seq);
2517
2518 /* TBD: notify client waiting for available dmem */
2519
2520 gk20a_dbg_fn("done");
2521
2522 return 0;
2523}
2524
2525static int pmu_wait_message_cond(struct pmu_gk20a *pmu, u32 timeout,
2526 u32 *var, u32 val);
2527
2528static void pmu_handle_zbc_msg(struct gk20a *g, struct pmu_msg *msg,
2529 void *param, u32 handle, u32 status)
2530{
2531 struct pmu_gk20a *pmu = param;
2532 pmu->zbc_save_done = 1;
2533}
2534
2535static void pmu_save_zbc(struct gk20a *g, u32 entries)
2536{
2537 struct pmu_gk20a *pmu = &g->pmu;
2538 struct pmu_cmd cmd;
2539 u32 seq;
2540
2541 if (!pmu->pmu_ready || !entries || !pmu->zbc_ready)
2542 return;
2543
2544 memset(&cmd, 0, sizeof(struct pmu_cmd));
2545 cmd.hdr.unit_id = PMU_UNIT_PG;
2546 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_zbc_cmd);
2547 cmd.cmd.zbc.cmd_type = g->ops.pmu_ver.cmd_id_zbc_table_update;
2548 cmd.cmd.zbc.entry_mask = ZBC_MASK(entries);
2549
2550 pmu->zbc_save_done = 0;
2551
2552 gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
2553 pmu_handle_zbc_msg, pmu, &seq, ~0);
2554 pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
2555 &pmu->zbc_save_done, 1);
2556 if (!pmu->zbc_save_done)
2557 gk20a_err(dev_from_gk20a(g), "ZBC save timeout");
2558}
2559
2560void gk20a_pmu_save_zbc(struct gk20a *g, u32 entries)
2561{
2562 if (g->pmu.zbc_ready)
2563 pmu_save_zbc(g, entries);
2564}
2565
2566static int pmu_perfmon_start_sampling(struct pmu_gk20a *pmu)
2567{
2568 struct gk20a *g = pmu->g;
2569 struct pmu_v *pv = &g->ops.pmu_ver;
2570 struct pmu_cmd cmd;
2571 struct pmu_payload payload;
2572 u32 current_rate = 0;
2573 u32 seq;
2574
2575 /* PERFMON Start */
2576 memset(&cmd, 0, sizeof(struct pmu_cmd));
2577 cmd.hdr.unit_id = PMU_UNIT_PERFMON;
2578 cmd.hdr.size = PMU_CMD_HDR_SIZE + pv->get_pmu_perfmon_cmd_start_size();
2579 pv->perfmon_start_set_cmd_type(&cmd.cmd.perfmon,
2580 PMU_PERFMON_CMD_ID_START);
2581 pv->perfmon_start_set_group_id(&cmd.cmd.perfmon,
2582 PMU_DOMAIN_GROUP_PSTATE);
2583 pv->perfmon_start_set_state_id(&cmd.cmd.perfmon,
2584 pmu->perfmon_state_id[PMU_DOMAIN_GROUP_PSTATE]);
2585
2586 current_rate = rate_gpu_to_gpc2clk(gk20a_clk_get_rate(g));
2587 if (current_rate >= gpc_pll_params.max_freq)
2588 pv->perfmon_start_set_flags(&cmd.cmd.perfmon,
2589 PMU_PERFMON_FLAG_ENABLE_DECREASE);
2590 else if (current_rate <= gpc_pll_params.min_freq)
2591 pv->perfmon_start_set_flags(&cmd.cmd.perfmon,
2592 PMU_PERFMON_FLAG_ENABLE_INCREASE);
2593 else
2594 pv->perfmon_start_set_flags(&cmd.cmd.perfmon,
2595 PMU_PERFMON_FLAG_ENABLE_INCREASE |
2596 PMU_PERFMON_FLAG_ENABLE_DECREASE);
2597
2598 pv->perfmon_start_set_flags(&cmd.cmd.perfmon,
2599 pv->perfmon_start_get_flags(&cmd.cmd.perfmon) |
2600 PMU_PERFMON_FLAG_CLEAR_PREV);
2601
2602 memset(&payload, 0, sizeof(struct pmu_payload));
2603
2604 /* TBD: PMU_PERFMON_PCT_TO_INC * 100 */
2605 pmu->perfmon_counter.upper_threshold = 3000; /* 30% */
2606 /* TBD: PMU_PERFMON_PCT_TO_DEC * 100 */
2607 pmu->perfmon_counter.lower_threshold = 1000; /* 10% */
2608 pmu->perfmon_counter.valid = true;
2609
2610 payload.in.buf = &pmu->perfmon_counter;
2611 payload.in.size = sizeof(pmu->perfmon_counter);
2612 payload.in.offset =
2613 pv->get_perfmon_cmd_start_offsetofvar(COUNTER_ALLOC);
2614
2615 gk20a_pmu_cmd_post(g, &cmd, NULL, &payload, PMU_COMMAND_QUEUE_LPQ,
2616 NULL, NULL, &seq, ~0);
2617
2618 return 0;
2619}
2620
2621static int pmu_perfmon_stop_sampling(struct pmu_gk20a *pmu)
2622{
2623 struct gk20a *g = pmu->g;
2624 struct pmu_cmd cmd;
2625 u32 seq;
2626
2627 /* PERFMON Stop */
2628 memset(&cmd, 0, sizeof(struct pmu_cmd));
2629 cmd.hdr.unit_id = PMU_UNIT_PERFMON;
2630 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_perfmon_cmd_stop);
2631 cmd.cmd.perfmon.stop.cmd_type = PMU_PERFMON_CMD_ID_STOP;
2632
2633 gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_LPQ,
2634 NULL, NULL, &seq, ~0);
2635 return 0;
2636}
2637
2638static int pmu_handle_perfmon_event(struct pmu_gk20a *pmu,
2639 struct pmu_perfmon_msg *msg)
2640{
2641 struct gk20a *g = pmu->g;
2642 u32 rate;
2643
2644 gk20a_dbg_fn("");
2645
2646 switch (msg->msg_type) {
2647 case PMU_PERFMON_MSG_ID_INCREASE_EVENT:
2648 gk20a_dbg_pmu("perfmon increase event: "
2649 "state_id %d, ground_id %d, pct %d",
2650 msg->gen.state_id, msg->gen.group_id, msg->gen.data);
2651 /* increase gk20a clock freq by 20% */
2652 rate = gk20a_clk_get_rate(g);
2653 gk20a_clk_set_rate(g, rate * 6 / 5);
2654 break;
2655 case PMU_PERFMON_MSG_ID_DECREASE_EVENT:
2656 gk20a_dbg_pmu("perfmon decrease event: "
2657 "state_id %d, ground_id %d, pct %d",
2658 msg->gen.state_id, msg->gen.group_id, msg->gen.data);
2659 /* decrease gk20a clock freq by 10% */
2660 rate = gk20a_clk_get_rate(g);
2661 gk20a_clk_set_rate(g, (rate / 10) * 7);
2662 break;
2663 case PMU_PERFMON_MSG_ID_INIT_EVENT:
2664 pmu->perfmon_ready = 1;
2665 gk20a_dbg_pmu("perfmon init event");
2666 break;
2667 default:
2668 break;
2669 }
2670
2671 /* restart sampling */
2672 if (IS_ENABLED(CONFIG_GK20A_PERFMON))
2673 return pmu_perfmon_start_sampling(pmu);
2674 return 0;
2675}
2676
2677
2678static int pmu_handle_event(struct pmu_gk20a *pmu, struct pmu_msg *msg)
2679{
2680 int err;
2681
2682 gk20a_dbg_fn("");
2683
2684 switch (msg->hdr.unit_id) {
2685 case PMU_UNIT_PERFMON:
2686 err = pmu_handle_perfmon_event(pmu, &msg->msg.perfmon);
2687 break;
2688 default:
2689 break;
2690 }
2691
2692 return err;
2693}
2694
2695static int pmu_process_message(struct pmu_gk20a *pmu)
2696{
2697 struct pmu_msg msg;
2698 int status;
2699
2700 if (unlikely(!pmu->pmu_ready)) {
2701 pmu_process_init_msg(pmu, &msg);
2702 pmu_init_powergating(pmu);
2703 pmu_init_perfmon(pmu);
2704 return 0;
2705 }
2706
2707 while (pmu_read_message(pmu,
2708 &pmu->queue[PMU_MESSAGE_QUEUE], &msg, &status)) {
2709
2710 gk20a_dbg_pmu("read msg hdr: "
2711 "unit_id = 0x%08x, size = 0x%08x, "
2712 "ctrl_flags = 0x%08x, seq_id = 0x%08x",
2713 msg.hdr.unit_id, msg.hdr.size,
2714 msg.hdr.ctrl_flags, msg.hdr.seq_id);
2715
2716 msg.hdr.ctrl_flags &= ~PMU_CMD_FLAGS_PMU_MASK;
2717
2718 if (msg.hdr.ctrl_flags == PMU_CMD_FLAGS_EVENT) {
2719 pmu_handle_event(pmu, &msg);
2720 } else {
2721 pmu_response_handle(pmu, &msg);
2722 }
2723 }
2724
2725 return 0;
2726}
2727
2728static int pmu_wait_message_cond(struct pmu_gk20a *pmu, u32 timeout,
2729 u32 *var, u32 val)
2730{
2731 struct gk20a *g = pmu->g;
2732 unsigned long end_jiffies = jiffies + msecs_to_jiffies(timeout);
2733 unsigned long delay = GR_IDLE_CHECK_DEFAULT;
2734
2735 do {
2736 if (*var == val)
2737 return 0;
2738
2739 if (gk20a_readl(g, pwr_falcon_irqstat_r()))
2740 gk20a_pmu_isr(g);
2741
2742 usleep_range(delay, delay * 2);
2743 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
2744 } while (time_before(jiffies, end_jiffies) ||
2745 !tegra_platform_is_silicon());
2746
2747 return -ETIMEDOUT;
2748}
2749
2750static void pmu_dump_elpg_stats(struct pmu_gk20a *pmu)
2751{
2752 struct gk20a *g = pmu->g;
2753 struct pmu_pg_stats stats;
2754
2755 pmu_copy_from_dmem(pmu, pmu->stat_dmem_offset,
2756 (u8 *)&stats, sizeof(struct pmu_pg_stats), 0);
2757
2758 gk20a_dbg_pmu("pg_entry_start_timestamp : 0x%016llx",
2759 stats.pg_entry_start_timestamp);
2760 gk20a_dbg_pmu("pg_exit_start_timestamp : 0x%016llx",
2761 stats.pg_exit_start_timestamp);
2762 gk20a_dbg_pmu("pg_ingating_start_timestamp : 0x%016llx",
2763 stats.pg_ingating_start_timestamp);
2764 gk20a_dbg_pmu("pg_ungating_start_timestamp : 0x%016llx",
2765 stats.pg_ungating_start_timestamp);
2766 gk20a_dbg_pmu("pg_avg_entry_time_us : 0x%08x",
2767 stats.pg_avg_entry_time_us);
2768 gk20a_dbg_pmu("pg_avg_exit_time_us : 0x%08x",
2769 stats.pg_avg_exit_time_us);
2770 gk20a_dbg_pmu("pg_ingating_cnt : 0x%08x",
2771 stats.pg_ingating_cnt);
2772 gk20a_dbg_pmu("pg_ingating_time_us : 0x%08x",
2773 stats.pg_ingating_time_us);
2774 gk20a_dbg_pmu("pg_ungating_count : 0x%08x",
2775 stats.pg_ungating_count);
2776 gk20a_dbg_pmu("pg_ungating_time_us 0x%08x: ",
2777 stats.pg_ungating_time_us);
2778 gk20a_dbg_pmu("pg_gating_cnt : 0x%08x",
2779 stats.pg_gating_cnt);
2780 gk20a_dbg_pmu("pg_gating_deny_cnt : 0x%08x",
2781 stats.pg_gating_deny_cnt);
2782
2783 /*
2784 Turn on PG_DEBUG in ucode and locate symbol "ElpgLog" offset
2785 in .nm file, e.g. 0x1000066c. use 0x66c.
2786 u32 i, val[20];
2787 pmu_copy_from_dmem(pmu, 0x66c,
2788 (u8 *)val, sizeof(val), 0);
2789 gk20a_dbg_pmu("elpg log begin");
2790 for (i = 0; i < 20; i++)
2791 gk20a_dbg_pmu("0x%08x", val[i]);
2792 gk20a_dbg_pmu("elpg log end");
2793 */
2794
2795 gk20a_dbg_pmu("pwr_pmu_idle_mask_supp_r(3): 0x%08x",
2796 gk20a_readl(g, pwr_pmu_idle_mask_supp_r(3)));
2797 gk20a_dbg_pmu("pwr_pmu_idle_mask_1_supp_r(3): 0x%08x",
2798 gk20a_readl(g, pwr_pmu_idle_mask_1_supp_r(3)));
2799 gk20a_dbg_pmu("pwr_pmu_idle_ctrl_supp_r(3): 0x%08x",
2800 gk20a_readl(g, pwr_pmu_idle_ctrl_supp_r(3)));
2801 gk20a_dbg_pmu("pwr_pmu_pg_idle_cnt_r(0): 0x%08x",
2802 gk20a_readl(g, pwr_pmu_pg_idle_cnt_r(0)));
2803 gk20a_dbg_pmu("pwr_pmu_pg_intren_r(0): 0x%08x",
2804 gk20a_readl(g, pwr_pmu_pg_intren_r(0)));
2805
2806 gk20a_dbg_pmu("pwr_pmu_idle_count_r(3): 0x%08x",
2807 gk20a_readl(g, pwr_pmu_idle_count_r(3)));
2808 gk20a_dbg_pmu("pwr_pmu_idle_count_r(4): 0x%08x",
2809 gk20a_readl(g, pwr_pmu_idle_count_r(4)));
2810 gk20a_dbg_pmu("pwr_pmu_idle_count_r(7): 0x%08x",
2811 gk20a_readl(g, pwr_pmu_idle_count_r(7)));
2812
2813 /*
2814 TBD: script can't generate those registers correctly
2815 gk20a_dbg_pmu("pwr_pmu_idle_status_r(): 0x%08x",
2816 gk20a_readl(g, pwr_pmu_idle_status_r()));
2817 gk20a_dbg_pmu("pwr_pmu_pg_ctrl_r(): 0x%08x",
2818 gk20a_readl(g, pwr_pmu_pg_ctrl_r()));
2819 */
2820}
2821
2822static void pmu_dump_falcon_stats(struct pmu_gk20a *pmu)
2823{
2824 struct gk20a *g = pmu->g;
2825 int i;
2826
2827 gk20a_err(dev_from_gk20a(g), "pwr_falcon_os_r : %d",
2828 gk20a_readl(g, pwr_falcon_os_r()));
2829 gk20a_err(dev_from_gk20a(g), "pwr_falcon_cpuctl_r : 0x%x",
2830 gk20a_readl(g, pwr_falcon_cpuctl_r()));
2831 gk20a_err(dev_from_gk20a(g), "pwr_falcon_idlestate_r : 0x%x",
2832 gk20a_readl(g, pwr_falcon_idlestate_r()));
2833 gk20a_err(dev_from_gk20a(g), "pwr_falcon_mailbox0_r : 0x%x",
2834 gk20a_readl(g, pwr_falcon_mailbox0_r()));
2835 gk20a_err(dev_from_gk20a(g), "pwr_falcon_mailbox1_r : 0x%x",
2836 gk20a_readl(g, pwr_falcon_mailbox1_r()));
2837 gk20a_err(dev_from_gk20a(g), "pwr_falcon_irqstat_r : 0x%x",
2838 gk20a_readl(g, pwr_falcon_irqstat_r()));
2839 gk20a_err(dev_from_gk20a(g), "pwr_falcon_irqmode_r : 0x%x",
2840 gk20a_readl(g, pwr_falcon_irqmode_r()));
2841 gk20a_err(dev_from_gk20a(g), "pwr_falcon_irqmask_r : 0x%x",
2842 gk20a_readl(g, pwr_falcon_irqmask_r()));
2843 gk20a_err(dev_from_gk20a(g), "pwr_falcon_irqdest_r : 0x%x",
2844 gk20a_readl(g, pwr_falcon_irqdest_r()));
2845
2846 for (i = 0; i < pwr_pmu_mailbox__size_1_v(); i++)
2847 gk20a_err(dev_from_gk20a(g), "pwr_pmu_mailbox_r(%d) : 0x%x",
2848 i, gk20a_readl(g, pwr_pmu_mailbox_r(i)));
2849
2850 for (i = 0; i < pwr_pmu_debug__size_1_v(); i++)
2851 gk20a_err(dev_from_gk20a(g), "pwr_pmu_debug_r(%d) : 0x%x",
2852 i, gk20a_readl(g, pwr_pmu_debug_r(i)));
2853
2854 for (i = 0; i < 6/*NV_PPWR_FALCON_ICD_IDX_RSTAT__SIZE_1*/; i++) {
2855 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2856 pwr_pmu_falcon_icd_cmd_opc_rstat_f() |
2857 pwr_pmu_falcon_icd_cmd_idx_f(i));
2858 gk20a_err(dev_from_gk20a(g), "pmu_rstat (%d) : 0x%x",
2859 i, gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2860 }
2861
2862 i = gk20a_readl(g, pwr_pmu_bar0_error_status_r());
2863 gk20a_err(dev_from_gk20a(g), "pwr_pmu_bar0_error_status_r : 0x%x", i);
2864 if (i != 0) {
2865 gk20a_err(dev_from_gk20a(g), "pwr_pmu_bar0_addr_r : 0x%x",
2866 gk20a_readl(g, pwr_pmu_bar0_addr_r()));
2867 gk20a_err(dev_from_gk20a(g), "pwr_pmu_bar0_data_r : 0x%x",
2868 gk20a_readl(g, pwr_pmu_bar0_data_r()));
2869 gk20a_err(dev_from_gk20a(g), "pwr_pmu_bar0_timeout_r : 0x%x",
2870 gk20a_readl(g, pwr_pmu_bar0_timeout_r()));
2871 gk20a_err(dev_from_gk20a(g), "pwr_pmu_bar0_ctl_r : 0x%x",
2872 gk20a_readl(g, pwr_pmu_bar0_ctl_r()));
2873 }
2874
2875 i = gk20a_readl(g, pwr_pmu_bar0_fecs_error_r());
2876 gk20a_err(dev_from_gk20a(g), "pwr_pmu_bar0_fecs_error_r : 0x%x", i);
2877
2878 i = gk20a_readl(g, pwr_falcon_exterrstat_r());
2879 gk20a_err(dev_from_gk20a(g), "pwr_falcon_exterrstat_r : 0x%x", i);
2880 if (pwr_falcon_exterrstat_valid_v(i) ==
2881 pwr_falcon_exterrstat_valid_true_v()) {
2882 gk20a_err(dev_from_gk20a(g), "pwr_falcon_exterraddr_r : 0x%x",
2883 gk20a_readl(g, pwr_falcon_exterraddr_r()));
2884 gk20a_err(dev_from_gk20a(g), "top_fs_status_r : 0x%x",
2885 gk20a_readl(g, top_fs_status_r()));
2886 gk20a_err(dev_from_gk20a(g), "pmc_enable : 0x%x",
2887 gk20a_readl(g, mc_enable_r()));
2888 }
2889
2890 gk20a_err(dev_from_gk20a(g), "pwr_falcon_engctl_r : 0x%x",
2891 gk20a_readl(g, pwr_falcon_engctl_r()));
2892 gk20a_err(dev_from_gk20a(g), "pwr_falcon_curctx_r : 0x%x",
2893 gk20a_readl(g, pwr_falcon_curctx_r()));
2894 gk20a_err(dev_from_gk20a(g), "pwr_falcon_nxtctx_r : 0x%x",
2895 gk20a_readl(g, pwr_falcon_nxtctx_r()));
2896
2897 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2898 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2899 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_IMB));
2900 gk20a_err(dev_from_gk20a(g), "PMU_FALCON_REG_IMB : 0x%x",
2901 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2902
2903 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2904 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2905 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_DMB));
2906 gk20a_err(dev_from_gk20a(g), "PMU_FALCON_REG_DMB : 0x%x",
2907 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2908
2909 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2910 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2911 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CSW));
2912 gk20a_err(dev_from_gk20a(g), "PMU_FALCON_REG_CSW : 0x%x",
2913 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2914
2915 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2916 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2917 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_CTX));
2918 gk20a_err(dev_from_gk20a(g), "PMU_FALCON_REG_CTX : 0x%x",
2919 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2920
2921 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2922 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2923 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_EXCI));
2924 gk20a_err(dev_from_gk20a(g), "PMU_FALCON_REG_EXCI : 0x%x",
2925 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2926
2927 for (i = 0; i < 4; i++) {
2928 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2929 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2930 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_PC));
2931 gk20a_err(dev_from_gk20a(g), "PMU_FALCON_REG_PC : 0x%x",
2932 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2933
2934 gk20a_writel(g, pwr_pmu_falcon_icd_cmd_r(),
2935 pwr_pmu_falcon_icd_cmd_opc_rreg_f() |
2936 pwr_pmu_falcon_icd_cmd_idx_f(PMU_FALCON_REG_SP));
2937 gk20a_err(dev_from_gk20a(g), "PMU_FALCON_REG_SP : 0x%x",
2938 gk20a_readl(g, pwr_pmu_falcon_icd_rdata_r()));
2939 }
2940
2941 /* PMU may crash due to FECS crash. Dump FECS status */
2942 gk20a_fecs_dump_falcon_stats(g);
2943}
2944
2945void gk20a_pmu_isr(struct gk20a *g)
2946{
2947 struct pmu_gk20a *pmu = &g->pmu;
2948 struct pmu_queue *queue;
2949 u32 intr, mask;
2950 bool recheck = false;
2951
2952 gk20a_dbg_fn("");
2953
2954 mutex_lock(&pmu->isr_mutex);
2955
2956 mask = gk20a_readl(g, pwr_falcon_irqmask_r()) &
2957 gk20a_readl(g, pwr_falcon_irqdest_r());
2958
2959 intr = gk20a_readl(g, pwr_falcon_irqstat_r()) & mask;
2960
2961 gk20a_dbg_pmu("received falcon interrupt: 0x%08x", intr);
2962
2963 if (!intr) {
2964 mutex_unlock(&pmu->isr_mutex);
2965 return;
2966 }
2967
2968 if (intr & pwr_falcon_irqstat_halt_true_f()) {
2969 gk20a_err(dev_from_gk20a(g),
2970 "pmu halt intr not implemented");
2971 pmu_dump_falcon_stats(pmu);
2972 }
2973 if (intr & pwr_falcon_irqstat_exterr_true_f()) {
2974 gk20a_err(dev_from_gk20a(g),
2975 "pmu exterr intr not implemented. Clearing interrupt.");
2976 pmu_dump_falcon_stats(pmu);
2977
2978 gk20a_writel(g, pwr_falcon_exterrstat_r(),
2979 gk20a_readl(g, pwr_falcon_exterrstat_r()) &
2980 ~pwr_falcon_exterrstat_valid_m());
2981 }
2982 if (intr & pwr_falcon_irqstat_swgen0_true_f()) {
2983 pmu_process_message(pmu);
2984 recheck = true;
2985 }
2986
2987 gk20a_writel(g, pwr_falcon_irqsclr_r(), intr);
2988
2989 if (recheck) {
2990 queue = &pmu->queue[PMU_MESSAGE_QUEUE];
2991 if (!pmu_queue_is_empty(pmu, queue))
2992 gk20a_writel(g, pwr_falcon_irqsset_r(),
2993 pwr_falcon_irqsset_swgen0_set_f());
2994 }
2995
2996 mutex_unlock(&pmu->isr_mutex);
2997}
2998
2999static bool pmu_validate_cmd(struct pmu_gk20a *pmu, struct pmu_cmd *cmd,
3000 struct pmu_msg *msg, struct pmu_payload *payload,
3001 u32 queue_id)
3002{
3003 struct gk20a *g = pmu->g;
3004 struct pmu_queue *queue;
3005 u32 in_size, out_size;
3006
3007 if (!PMU_IS_SW_COMMAND_QUEUE(queue_id))
3008 goto invalid_cmd;
3009
3010 queue = &pmu->queue[queue_id];
3011 if (cmd->hdr.size < PMU_CMD_HDR_SIZE)
3012 goto invalid_cmd;
3013
3014 if (cmd->hdr.size > (queue->size >> 1))
3015 goto invalid_cmd;
3016
3017 if (msg != NULL && msg->hdr.size < PMU_MSG_HDR_SIZE)
3018 goto invalid_cmd;
3019
3020 if (!PMU_UNIT_ID_IS_VALID(cmd->hdr.unit_id))
3021 goto invalid_cmd;
3022
3023 if (payload == NULL)
3024 return true;
3025
3026 if (payload->in.buf == NULL && payload->out.buf == NULL)
3027 goto invalid_cmd;
3028
3029 if ((payload->in.buf != NULL && payload->in.size == 0) ||
3030 (payload->out.buf != NULL && payload->out.size == 0))
3031 goto invalid_cmd;
3032
3033 in_size = PMU_CMD_HDR_SIZE;
3034 if (payload->in.buf) {
3035 in_size += payload->in.offset;
3036 in_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu);
3037 }
3038
3039 out_size = PMU_CMD_HDR_SIZE;
3040 if (payload->out.buf) {
3041 out_size += payload->out.offset;
3042 out_size += g->ops.pmu_ver.get_pmu_allocation_struct_size(pmu);
3043 }
3044
3045 if (in_size > cmd->hdr.size || out_size > cmd->hdr.size)
3046 goto invalid_cmd;
3047
3048
3049 if ((payload->in.offset != 0 && payload->in.buf == NULL) ||
3050 (payload->out.offset != 0 && payload->out.buf == NULL))
3051 goto invalid_cmd;
3052
3053 return true;
3054
3055invalid_cmd:
3056 gk20a_err(dev_from_gk20a(g), "invalid pmu cmd :\n"
3057 "queue_id=%d,\n"
3058 "cmd_size=%d, cmd_unit_id=%d, msg=%p, msg_size=%d,\n"
3059 "payload in=%p, in_size=%d, in_offset=%d,\n"
3060 "payload out=%p, out_size=%d, out_offset=%d",
3061 queue_id, cmd->hdr.size, cmd->hdr.unit_id,
3062 msg, msg?msg->hdr.unit_id:~0,
3063 &payload->in, payload->in.size, payload->in.offset,
3064 &payload->out, payload->out.size, payload->out.offset);
3065
3066 return false;
3067}
3068
3069static int pmu_write_cmd(struct pmu_gk20a *pmu, struct pmu_cmd *cmd,
3070 u32 queue_id, unsigned long timeout)
3071{
3072 struct gk20a *g = pmu->g;
3073 struct pmu_queue *queue;
3074 unsigned long end_jiffies = jiffies +
3075 msecs_to_jiffies(timeout);
3076 int err;
3077
3078 gk20a_dbg_fn("");
3079
3080 queue = &pmu->queue[queue_id];
3081
3082 do {
3083 err = pmu_queue_open_write(pmu, queue, cmd->hdr.size);
3084 if (err == -EAGAIN && time_before(jiffies, end_jiffies))
3085 usleep_range(1000, 2000);
3086 else
3087 break;
3088 } while (1);
3089
3090 if (err)
3091 goto clean_up;
3092
3093 pmu_queue_push(pmu, queue, cmd, cmd->hdr.size);
3094
3095 err = pmu_queue_close(pmu, queue, true);
3096
3097clean_up:
3098 if (err)
3099 gk20a_err(dev_from_gk20a(g),
3100 "fail to write cmd to queue %d", queue_id);
3101 else
3102 gk20a_dbg_fn("done");
3103
3104 return err;
3105}
3106
3107int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd,
3108 struct pmu_msg *msg, struct pmu_payload *payload,
3109 u32 queue_id, pmu_callback callback, void* cb_param,
3110 u32 *seq_desc, unsigned long timeout)
3111{
3112 struct pmu_gk20a *pmu = &g->pmu;
3113 struct pmu_v *pv = &g->ops.pmu_ver;
3114 struct pmu_sequence *seq;
3115 void *in = NULL, *out = NULL;
3116 int err;
3117
3118 gk20a_dbg_fn("");
3119
3120 BUG_ON(!cmd);
3121 BUG_ON(!seq_desc);
3122 BUG_ON(!pmu->pmu_ready);
3123
3124 if (!pmu_validate_cmd(pmu, cmd, msg, payload, queue_id))
3125 return -EINVAL;
3126
3127 err = pmu_seq_acquire(pmu, &seq);
3128 if (err)
3129 return err;
3130
3131 cmd->hdr.seq_id = seq->id;
3132
3133 cmd->hdr.ctrl_flags = 0;
3134 cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_STATUS;
3135 cmd->hdr.ctrl_flags |= PMU_CMD_FLAGS_INTR;
3136
3137 seq->callback = callback;
3138 seq->cb_params = cb_param;
3139 seq->msg = msg;
3140 seq->out_payload = NULL;
3141 seq->desc = pmu->next_seq_desc++;
3142
3143 if (payload)
3144 seq->out_payload = payload->out.buf;
3145
3146 *seq_desc = seq->desc;
3147
3148 if (payload && payload->in.offset != 0) {
3149 pv->set_pmu_allocation_ptr(pmu, &in,
3150 ((u8 *)&cmd->cmd + payload->in.offset));
3151
3152 if (payload->in.buf != payload->out.buf)
3153 pv->pmu_allocation_set_dmem_size(pmu, in,
3154 (u16)payload->in.size);
3155 else
3156 pv->pmu_allocation_set_dmem_size(pmu, in,
3157 (u16)max(payload->in.size, payload->out.size));
3158
3159 err = pmu->dmem.alloc(&pmu->dmem,
3160 pv->pmu_allocation_get_dmem_offset_addr(pmu, in),
3161 pv->pmu_allocation_get_dmem_size(pmu, in));
3162 if (err)
3163 goto clean_up;
3164
3165 pmu_copy_to_dmem(pmu, (pv->pmu_allocation_get_dmem_offset(pmu,
3166 in)),
3167 payload->in.buf, payload->in.size, 0);
3168 pv->pmu_allocation_set_dmem_size(pmu,
3169 pv->get_pmu_seq_in_a_ptr(seq),
3170 pv->pmu_allocation_get_dmem_size(pmu, in));
3171 pv->pmu_allocation_set_dmem_offset(pmu,
3172 pv->get_pmu_seq_in_a_ptr(seq),
3173 pv->pmu_allocation_get_dmem_offset(pmu, in));
3174 }
3175
3176 if (payload && payload->out.offset != 0) {
3177 pv->set_pmu_allocation_ptr(pmu, &out,
3178 ((u8 *)&cmd->cmd + payload->out.offset));
3179 pv->pmu_allocation_set_dmem_size(pmu, out,
3180 (u16)payload->out.size);
3181
3182 if (payload->out.buf != payload->in.buf) {
3183 err = pmu->dmem.alloc(&pmu->dmem,
3184 pv->pmu_allocation_get_dmem_offset_addr(pmu, out),
3185 pv->pmu_allocation_get_dmem_size(pmu, out));
3186 if (err)
3187 goto clean_up;
3188 } else {
3189 BUG_ON(in == NULL);
3190 pv->pmu_allocation_set_dmem_offset(pmu, out,
3191 pv->pmu_allocation_get_dmem_offset(pmu, in));
3192 }
3193
3194 pv->pmu_allocation_set_dmem_size(pmu,
3195 pv->get_pmu_seq_out_a_ptr(seq),
3196 pv->pmu_allocation_get_dmem_size(pmu, out));
3197 pv->pmu_allocation_set_dmem_offset(pmu,
3198 pv->get_pmu_seq_out_a_ptr(seq),
3199 pv->pmu_allocation_get_dmem_offset(pmu, out));
3200 }
3201
3202 seq->state = PMU_SEQ_STATE_USED;
3203 err = pmu_write_cmd(pmu, cmd, queue_id, timeout);
3204 if (err)
3205 seq->state = PMU_SEQ_STATE_PENDING;
3206
3207 gk20a_dbg_fn("done");
3208
3209 return 0;
3210
3211clean_up:
3212 gk20a_dbg_fn("fail");
3213 if (in)
3214 pmu->dmem.free(&pmu->dmem,
3215 pv->pmu_allocation_get_dmem_offset(pmu, in),
3216 pv->pmu_allocation_get_dmem_size(pmu, in));
3217 if (out)
3218 pmu->dmem.free(&pmu->dmem,
3219 pv->pmu_allocation_get_dmem_offset(pmu, out),
3220 pv->pmu_allocation_get_dmem_size(pmu, out));
3221
3222 pmu_seq_release(pmu, seq);
3223 return err;
3224}
3225
3226static int gk20a_pmu_enable_elpg_locked(struct gk20a *g)
3227{
3228 struct pmu_gk20a *pmu = &g->pmu;
3229 struct pmu_cmd cmd;
3230 u32 seq, status;
3231
3232 gk20a_dbg_fn("");
3233
3234 memset(&cmd, 0, sizeof(struct pmu_cmd));
3235 cmd.hdr.unit_id = PMU_UNIT_PG;
3236 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
3237 cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
3238 cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
3239 cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_ALLOW;
3240
3241 /* no need to wait ack for ELPG enable but set pending to sync
3242 with follow up ELPG disable */
3243 pmu->elpg_stat = PMU_ELPG_STAT_ON_PENDING;
3244
3245 status = gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
3246 pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
3247
3248 BUG_ON(status != 0);
3249
3250 gk20a_dbg_fn("done");
3251 return 0;
3252}
3253
3254int gk20a_pmu_enable_elpg(struct gk20a *g)
3255{
3256 struct pmu_gk20a *pmu = &g->pmu;
3257 struct gr_gk20a *gr = &g->gr;
3258
3259 int ret = 0;
3260
3261 gk20a_dbg_fn("");
3262
3263 if (!pmu->elpg_ready || !pmu->initialized)
3264 goto exit;
3265
3266 mutex_lock(&pmu->elpg_mutex);
3267
3268 pmu->elpg_refcnt++;
3269 if (pmu->elpg_refcnt <= 0)
3270 goto exit_unlock;
3271
3272 /* something is not right if we end up in following code path */
3273 if (unlikely(pmu->elpg_refcnt > 1)) {
3274 gk20a_warn(dev_from_gk20a(g),
3275 "%s(): possible elpg refcnt mismatch. elpg refcnt=%d",
3276 __func__, pmu->elpg_refcnt);
3277 WARN_ON(1);
3278 }
3279
3280 /* do NOT enable elpg until golden ctx is created,
3281 which is related with the ctx that ELPG save and restore. */
3282 if (unlikely(!gr->ctx_vars.golden_image_initialized))
3283 goto exit_unlock;
3284
3285 /* return if ELPG is already on or on_pending or off_on_pending */
3286 if (pmu->elpg_stat != PMU_ELPG_STAT_OFF)
3287 goto exit_unlock;
3288
3289 /* if ELPG is not allowed right now, mark that it should be enabled
3290 * immediately after it is allowed */
3291 if (!pmu->elpg_enable_allow) {
3292 pmu->elpg_stat = PMU_ELPG_STAT_OFF_ON_PENDING;
3293 goto exit_unlock;
3294 }
3295
3296 ret = gk20a_pmu_enable_elpg_locked(g);
3297
3298exit_unlock:
3299 mutex_unlock(&pmu->elpg_mutex);
3300exit:
3301 gk20a_dbg_fn("done");
3302 return ret;
3303}
3304
3305static void pmu_elpg_enable_allow(struct work_struct *work)
3306{
3307 struct pmu_gk20a *pmu = container_of(to_delayed_work(work),
3308 struct pmu_gk20a, elpg_enable);
3309
3310 gk20a_dbg_fn("");
3311
3312 mutex_lock(&pmu->elpg_mutex);
3313
3314 /* It is ok to enabled powergating now */
3315 pmu->elpg_enable_allow = true;
3316
3317 /* do we have pending requests? */
3318 if (pmu->elpg_stat == PMU_ELPG_STAT_OFF_ON_PENDING) {
3319 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
3320 gk20a_pmu_enable_elpg_locked(pmu->g);
3321 }
3322
3323 mutex_unlock(&pmu->elpg_mutex);
3324
3325 gk20a_dbg_fn("done");
3326}
3327
3328static int gk20a_pmu_disable_elpg_defer_enable(struct gk20a *g, bool enable)
3329{
3330 struct pmu_gk20a *pmu = &g->pmu;
3331 struct pmu_cmd cmd;
3332 u32 seq;
3333 int ret = 0;
3334
3335 gk20a_dbg_fn("");
3336
3337 if (!pmu->elpg_ready || !pmu->initialized)
3338 return 0;
3339
3340 /* remove the work from queue */
3341 cancel_delayed_work_sync(&pmu->elpg_enable);
3342
3343 mutex_lock(&pmu->elpg_mutex);
3344
3345 pmu->elpg_refcnt--;
3346 if (pmu->elpg_refcnt > 0) {
3347 gk20a_warn(dev_from_gk20a(g),
3348 "%s(): possible elpg refcnt mismatch. elpg refcnt=%d",
3349 __func__, pmu->elpg_refcnt);
3350 WARN_ON(1);
3351 ret = 0;
3352 goto exit_unlock;
3353 }
3354
3355 /* cancel off_on_pending and return */
3356 if (pmu->elpg_stat == PMU_ELPG_STAT_OFF_ON_PENDING) {
3357 pmu->elpg_stat = PMU_ELPG_STAT_OFF;
3358 ret = 0;
3359 goto exit_reschedule;
3360 }
3361 /* wait if on_pending */
3362 else if (pmu->elpg_stat == PMU_ELPG_STAT_ON_PENDING) {
3363
3364 pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
3365 &pmu->elpg_stat, PMU_ELPG_STAT_ON);
3366
3367 if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
3368 gk20a_err(dev_from_gk20a(g),
3369 "ELPG_ALLOW_ACK failed, elpg_stat=%d",
3370 pmu->elpg_stat);
3371 pmu_dump_elpg_stats(pmu);
3372 pmu_dump_falcon_stats(pmu);
3373 ret = -EBUSY;
3374 goto exit_unlock;
3375 }
3376 }
3377 /* return if ELPG is already off */
3378 else if (pmu->elpg_stat != PMU_ELPG_STAT_ON) {
3379 ret = 0;
3380 goto exit_reschedule;
3381 }
3382
3383 memset(&cmd, 0, sizeof(struct pmu_cmd));
3384 cmd.hdr.unit_id = PMU_UNIT_PG;
3385 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(struct pmu_pg_cmd_elpg_cmd);
3386 cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
3387 cmd.cmd.pg.elpg_cmd.engine_id = ENGINE_GR_GK20A;
3388 cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_DISALLOW;
3389
3390 pmu->elpg_stat = PMU_ELPG_STAT_OFF_PENDING;
3391
3392 gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
3393 pmu_handle_pg_elpg_msg, pmu, &seq, ~0);
3394
3395 pmu_wait_message_cond(pmu, gk20a_get_gr_idle_timeout(g),
3396 &pmu->elpg_stat, PMU_ELPG_STAT_OFF);
3397 if (pmu->elpg_stat != PMU_ELPG_STAT_OFF) {
3398 gk20a_err(dev_from_gk20a(g),
3399 "ELPG_DISALLOW_ACK failed");
3400 pmu_dump_elpg_stats(pmu);
3401 pmu_dump_falcon_stats(pmu);
3402 ret = -EBUSY;
3403 goto exit_unlock;
3404 }
3405
3406exit_reschedule:
3407 if (enable) {
3408 pmu->elpg_enable_allow = false;
3409 schedule_delayed_work(&pmu->elpg_enable,
3410 msecs_to_jiffies(PMU_ELPG_ENABLE_ALLOW_DELAY_MSEC));
3411 } else
3412 pmu->elpg_enable_allow = true;
3413
3414
3415exit_unlock:
3416 mutex_unlock(&pmu->elpg_mutex);
3417 gk20a_dbg_fn("done");
3418 return ret;
3419}
3420
3421int gk20a_pmu_disable_elpg(struct gk20a *g)
3422{
3423 return gk20a_pmu_disable_elpg_defer_enable(g, true);
3424}
3425
3426int gk20a_pmu_perfmon_enable(struct gk20a *g, bool enable)
3427{
3428 struct pmu_gk20a *pmu = &g->pmu;
3429 int err;
3430
3431 gk20a_dbg_fn("");
3432
3433 if (enable)
3434 err = pmu_perfmon_start_sampling(pmu);
3435 else
3436 err = pmu_perfmon_stop_sampling(pmu);
3437
3438 return err;
3439}
3440
3441int gk20a_pmu_destroy(struct gk20a *g)
3442{
3443 struct pmu_gk20a *pmu = &g->pmu;
3444 u32 elpg_ingating_time, elpg_ungating_time, gating_cnt;
3445
3446 gk20a_dbg_fn("");
3447
3448 if (!support_gk20a_pmu())
3449 return 0;
3450
3451 /* make sure the pending operations are finished before we continue */
3452 cancel_delayed_work_sync(&pmu->elpg_enable);
3453 cancel_work_sync(&pmu->pg_init);
3454
3455 gk20a_pmu_get_elpg_residency_gating(g, &elpg_ingating_time,
3456 &elpg_ungating_time, &gating_cnt);
3457
3458 gk20a_pmu_disable_elpg_defer_enable(g, false);
3459 pmu->initialized = false;
3460
3461 /* update the s/w ELPG residency counters */
3462 g->pg_ingating_time_us += (u64)elpg_ingating_time;
3463 g->pg_ungating_time_us += (u64)elpg_ungating_time;
3464 g->pg_gating_cnt += gating_cnt;
3465
3466 pmu_enable(pmu, false);
3467
3468 if (pmu->remove_support) {
3469 pmu->remove_support(pmu);
3470 pmu->remove_support = NULL;
3471 }
3472
3473 gk20a_dbg_fn("done");
3474 return 0;
3475}
3476
3477int gk20a_pmu_load_norm(struct gk20a *g, u32 *load)
3478{
3479 struct pmu_gk20a *pmu = &g->pmu;
3480 u16 _load = 0;
3481
3482 if (!pmu->perfmon_ready) {
3483 *load = 0;
3484 return 0;
3485 }
3486
3487 pmu_copy_from_dmem(pmu, pmu->sample_buffer, (u8 *)&_load, 2, 0);
3488 *load = _load / 10;
3489
3490 return 0;
3491}
3492
3493void gk20a_pmu_get_load_counters(struct gk20a *g, u32 *busy_cycles,
3494 u32 *total_cycles)
3495{
3496 if (!g->power_on) {
3497 *busy_cycles = 0;
3498 *total_cycles = 0;
3499 return;
3500 }
3501
3502 gk20a_busy(g->dev);
3503 *busy_cycles = pwr_pmu_idle_count_value_v(
3504 gk20a_readl(g, pwr_pmu_idle_count_r(1)));
3505 rmb();
3506 *total_cycles = pwr_pmu_idle_count_value_v(
3507 gk20a_readl(g, pwr_pmu_idle_count_r(2)));
3508 gk20a_idle(g->dev);
3509}
3510
3511void gk20a_pmu_reset_load_counters(struct gk20a *g)
3512{
3513 u32 reg_val = pwr_pmu_idle_count_reset_f(1);
3514
3515 if (!g->power_on)
3516 return;
3517
3518 gk20a_busy(g->dev);
3519 gk20a_writel(g, pwr_pmu_idle_count_r(2), reg_val);
3520 wmb();
3521 gk20a_writel(g, pwr_pmu_idle_count_r(1), reg_val);
3522 gk20a_idle(g->dev);
3523}
3524
3525static int gk20a_pmu_get_elpg_residency_gating(struct gk20a *g,
3526 u32 *ingating_time, u32 *ungating_time, u32 *gating_cnt)
3527{
3528 struct pmu_gk20a *pmu = &g->pmu;
3529 struct pmu_pg_stats stats;
3530
3531 if (!pmu->initialized) {
3532 *ingating_time = 0;
3533 *ungating_time = 0;
3534 *gating_cnt = 0;
3535 return 0;
3536 }
3537
3538 pmu_copy_from_dmem(pmu, pmu->stat_dmem_offset,
3539 (u8 *)&stats, sizeof(struct pmu_pg_stats), 0);
3540
3541 *ingating_time = stats.pg_ingating_time_us;
3542 *ungating_time = stats.pg_ungating_time_us;
3543 *gating_cnt = stats.pg_gating_cnt;
3544
3545 return 0;
3546}
3547
3548/* Send an Adaptive Power (AP) related command to PMU */
3549static int gk20a_pmu_ap_send_command(struct gk20a *g,
3550 union pmu_ap_cmd *p_ap_cmd, bool b_block)
3551{
3552 struct pmu_gk20a *pmu = &g->pmu;
3553 /* FIXME: where is the PG structure defined?? */
3554 u32 status = 0;
3555 struct pmu_cmd cmd;
3556 u32 seq;
3557 pmu_callback p_callback = NULL;
3558
3559 memset(&cmd, 0, sizeof(struct pmu_cmd));
3560
3561 /* Copy common members */
3562 cmd.hdr.unit_id = PMU_UNIT_PG;
3563 cmd.hdr.size = PMU_CMD_HDR_SIZE + sizeof(union pmu_ap_cmd);
3564
3565 cmd.cmd.pg.ap_cmd.cmn.cmd_type = PMU_PG_CMD_ID_AP;
3566 cmd.cmd.pg.ap_cmd.cmn.cmd_id = p_ap_cmd->cmn.cmd_id;
3567
3568 /* Copy other members of command */
3569 switch (p_ap_cmd->cmn.cmd_id) {
3570 case PMU_AP_CMD_ID_INIT:
3571 cmd.cmd.pg.ap_cmd.init.pg_sampling_period_us =
3572 p_ap_cmd->init.pg_sampling_period_us;
3573 p_callback = ap_callback_init_and_enable_ctrl;
3574 break;
3575
3576 case PMU_AP_CMD_ID_INIT_AND_ENABLE_CTRL:
3577 cmd.cmd.pg.ap_cmd.init_and_enable_ctrl.ctrl_id =
3578 p_ap_cmd->init_and_enable_ctrl.ctrl_id;
3579 memcpy(
3580 (void *)&(cmd.cmd.pg.ap_cmd.init_and_enable_ctrl.params),
3581 (void *)&(p_ap_cmd->init_and_enable_ctrl.params),
3582 sizeof(struct pmu_ap_ctrl_init_params));
3583
3584 p_callback = ap_callback_init_and_enable_ctrl;
3585 break;
3586
3587 case PMU_AP_CMD_ID_ENABLE_CTRL:
3588 cmd.cmd.pg.ap_cmd.enable_ctrl.ctrl_id =
3589 p_ap_cmd->enable_ctrl.ctrl_id;
3590 break;
3591
3592 case PMU_AP_CMD_ID_DISABLE_CTRL:
3593 cmd.cmd.pg.ap_cmd.disable_ctrl.ctrl_id =
3594 p_ap_cmd->disable_ctrl.ctrl_id;
3595 break;
3596
3597 case PMU_AP_CMD_ID_KICK_CTRL:
3598 cmd.cmd.pg.ap_cmd.kick_ctrl.ctrl_id =
3599 p_ap_cmd->kick_ctrl.ctrl_id;
3600 cmd.cmd.pg.ap_cmd.kick_ctrl.skip_count =
3601 p_ap_cmd->kick_ctrl.skip_count;
3602 break;
3603
3604 default:
3605 gk20a_dbg_pmu("%s: Invalid Adaptive Power command %d\n",
3606 __func__, p_ap_cmd->cmn.cmd_id);
3607 return 0x2f;
3608 }
3609
3610 status = gk20a_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
3611 p_callback, pmu, &seq, ~0);
3612
3613 if (!status) {
3614 gk20a_dbg_pmu(
3615 "%s: Unable to submit Adaptive Power Command %d\n",
3616 __func__, p_ap_cmd->cmn.cmd_id);
3617 goto err_return;
3618 }
3619
3620 /* TODO: Implement blocking calls (b_block) */
3621
3622err_return:
3623 return status;
3624}
3625
3626static void ap_callback_init_and_enable_ctrl(
3627 struct gk20a *g, struct pmu_msg *msg,
3628 void *param, u32 seq_desc, u32 status)
3629{
3630 /* Define p_ap (i.e pointer to pmu_ap structure) */
3631 WARN_ON(!msg);
3632
3633 if (!status) {
3634 switch (msg->msg.pg.ap_msg.cmn.msg_id) {
3635 case PMU_AP_MSG_ID_INIT_ACK:
3636 break;
3637
3638 default:
3639 gk20a_dbg_pmu(
3640 "%s: Invalid Adaptive Power Message: %x\n",
3641 __func__, msg->msg.pg.ap_msg.cmn.msg_id);
3642 break;
3643 }
3644 }
3645}
3646
3647static int gk20a_aelpg_init(struct gk20a *g)
3648{
3649 int status = 0;
3650
3651 /* Remove reliance on app_ctrl field. */
3652 union pmu_ap_cmd ap_cmd;
3653
3654 /* TODO: Check for elpg being ready? */
3655 ap_cmd.init.cmd_id = PMU_AP_CMD_ID_INIT;
3656 ap_cmd.init.pg_sampling_period_us =
3657 APCTRL_SAMPLING_PERIOD_PG_DEFAULT_US;
3658
3659 status = gk20a_pmu_ap_send_command(g, &ap_cmd, false);
3660 return status;
3661}
3662
3663static int gk20a_aelpg_init_and_enable(struct gk20a *g, u8 ctrl_id)
3664{
3665 int status = 0;
3666 union pmu_ap_cmd ap_cmd;
3667
3668 /* TODO: Probably check if ELPG is ready? */
3669
3670 ap_cmd.init_and_enable_ctrl.cmd_id = PMU_AP_CMD_ID_INIT_AND_ENABLE_CTRL;
3671 ap_cmd.init_and_enable_ctrl.ctrl_id = ctrl_id;
3672 ap_cmd.init_and_enable_ctrl.params.min_idle_filter_us =
3673 APCTRL_MINIMUM_IDLE_FILTER_DEFAULT_US;
3674 ap_cmd.init_and_enable_ctrl.params.min_target_saving_us =
3675 APCTRL_MINIMUM_TARGET_SAVING_DEFAULT_US;
3676 ap_cmd.init_and_enable_ctrl.params.power_break_even_us =
3677 APCTRL_POWER_BREAKEVEN_DEFAULT_US;
3678 ap_cmd.init_and_enable_ctrl.params.cycles_per_sample_max =
3679 APCTRL_CYCLES_PER_SAMPLE_MAX_DEFAULT;
3680
3681 switch (ctrl_id) {
3682 case PMU_AP_CTRL_ID_GRAPHICS:
3683 break;
3684 default:
3685 break;
3686 }
3687
3688 status = gk20a_pmu_ap_send_command(g, &ap_cmd, true);
3689 return status;
3690}
3691
3692#if CONFIG_DEBUG_FS
3693static int elpg_residency_show(struct seq_file *s, void *data)
3694{
3695 struct gk20a *g = s->private;
3696 u32 ingating_time = 0;
3697 u32 ungating_time = 0;
3698 u32 gating_cnt;
3699 u64 total_ingating, total_ungating, residency, divisor, dividend;
3700
3701 /* Don't unnecessarily power on the device */
3702 if (g->power_on) {
3703 gk20a_busy(g->dev);
3704 gk20a_pmu_get_elpg_residency_gating(g, &ingating_time,
3705 &ungating_time, &gating_cnt);
3706 gk20a_idle(g->dev);
3707 }
3708 total_ingating = g->pg_ingating_time_us + (u64)ingating_time;
3709 total_ungating = g->pg_ungating_time_us + (u64)ungating_time;
3710 divisor = total_ingating + total_ungating;
3711
3712 /* We compute the residency on a scale of 1000 */
3713 dividend = total_ingating * 1000;
3714
3715 if (divisor)
3716 residency = div64_u64(dividend, divisor);
3717 else
3718 residency = 0;
3719
3720 seq_printf(s, "Time in ELPG: %llu us\n"
3721 "Time out of ELPG: %llu us\n"
3722 "ELPG residency ratio: %llu\n",
3723 total_ingating, total_ungating, residency);
3724 return 0;
3725
3726}
3727
3728static int elpg_residency_open(struct inode *inode, struct file *file)
3729{
3730 return single_open(file, elpg_residency_show, inode->i_private);
3731}
3732
3733static const struct file_operations elpg_residency_fops = {
3734 .open = elpg_residency_open,
3735 .read = seq_read,
3736 .llseek = seq_lseek,
3737 .release = single_release,
3738};
3739
3740static int elpg_transitions_show(struct seq_file *s, void *data)
3741{
3742 struct gk20a *g = s->private;
3743 u32 ingating_time, ungating_time, total_gating_cnt;
3744 u32 gating_cnt = 0;
3745
3746 if (g->power_on) {
3747 gk20a_busy(g->dev);
3748 gk20a_pmu_get_elpg_residency_gating(g, &ingating_time,
3749 &ungating_time, &gating_cnt);
3750 gk20a_idle(g->dev);
3751 }
3752 total_gating_cnt = g->pg_gating_cnt + gating_cnt;
3753
3754 seq_printf(s, "%u\n", total_gating_cnt);
3755 return 0;
3756
3757}
3758
3759static int elpg_transitions_open(struct inode *inode, struct file *file)
3760{
3761 return single_open(file, elpg_transitions_show, inode->i_private);
3762}
3763
3764static const struct file_operations elpg_transitions_fops = {
3765 .open = elpg_transitions_open,
3766 .read = seq_read,
3767 .llseek = seq_lseek,
3768 .release = single_release,
3769};
3770
3771int gk20a_pmu_debugfs_init(struct platform_device *dev)
3772{
3773 struct dentry *d;
3774 struct gk20a_platform *platform = platform_get_drvdata(dev);
3775 struct gk20a *g = get_gk20a(dev);
3776
3777 d = debugfs_create_file(
3778 "elpg_residency", S_IRUGO|S_IWUSR, platform->debugfs, g,
3779 &elpg_residency_fops);
3780 if (!d)
3781 goto err_out;
3782
3783 d = debugfs_create_file(
3784 "elpg_transitions", S_IRUGO, platform->debugfs, g,
3785 &elpg_transitions_fops);
3786 if (!d)
3787 goto err_out;
3788
3789 return 0;
3790
3791err_out:
3792 pr_err("%s: Failed to make debugfs node\n", __func__);
3793 debugfs_remove_recursive(platform->debugfs);
3794 return -ENOMEM;
3795}
3796#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-04 20:56:43 -0400