summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/clk/clk_arb.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/clk/clk_arb.c')
-rw-r--r--drivers/gpu/nvgpu/clk/clk_arb.c1548
1 files changed, 1548 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/clk/clk_arb.c b/drivers/gpu/nvgpu/clk/clk_arb.c
new file mode 100644
index 00000000..c440dc3b
--- /dev/null
+++ b/drivers/gpu/nvgpu/clk/clk_arb.c
@@ -0,0 +1,1548 @@
1/*
2 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 */
13
14#include "gk20a/gk20a.h"
15
16#include <linux/cdev.h>
17#include <linux/file.h>
18#include <linux/anon_inodes.h>
19#include <linux/nvgpu.h>
20#include <linux/bitops.h>
21#include <linux/spinlock.h>
22#include <linux/rculist.h>
23#include <linux/llist.h>
24#include "clk/clk_arb.h"
25
26
27#define MAX_F_POINTS 127
28
29#ifdef CONFIG_DEBUG_FS
30static int nvgpu_clk_arb_debugfs_init(struct gk20a *g);
31#endif
32
33static int nvgpu_clk_arb_release_event_dev(struct inode *inode,
34 struct file *filp);
35static int nvgpu_clk_arb_release_completion_dev(struct inode *inode,
36 struct file *filp);
37static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait);
38
39static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work);
40static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work);
41static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb);
42static void nvgpu_clk_arb_free_fd(struct kref *refcount);
43static void nvgpu_clk_arb_free_session(struct kref *refcount);
44static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target,
45 u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv,
46 u32 voltuv_sram);
47static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb,
48 u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk,
49 u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram);
50
51#define VF_POINT_INVALID_PSTATE ~0U
52#define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b)))
53#define VF_POINT_GET_PSTATE(a) (((a)->pstates) ?\
54 __fls((a)->pstates) :\
55 VF_POINT_INVALID_PSTATE)
56#define VF_POINT_COMMON_PSTATE(a, b) (((a)->pstates & (b)->pstates) ?\
57 __fls((a)->pstates & (b)->pstates) :\
58 VF_POINT_INVALID_PSTATE)
59
60struct nvgpu_clk_vf_point {
61 u16 pstates;
62 union {
63 struct {
64 u16 gpc_mhz;
65 u16 sys_mhz;
66 u16 xbar_mhz;
67 };
68 u16 mem_mhz;
69 };
70 u32 uvolt;
71 u32 uvolt_sram;
72};
73
74struct nvgpu_clk_vf_table {
75 u32 mclk_num_points;
76 struct nvgpu_clk_vf_point *mclk_points;
77 u32 gpc2clk_num_points;
78 struct nvgpu_clk_vf_point *gpc2clk_points;
79};
80#ifdef CONFIG_DEBUG_FS
81struct nvgpu_clk_arb_debug {
82 s64 switch_max;
83 s64 switch_min;
84 u64 switch_num;
85 s64 switch_avg;
86 s64 switch_std;
87};
88#endif
89
90struct nvgpu_clk_arb_target {
91 u16 mclk;
92 u16 gpc2clk;
93 u32 pstate;
94};
95
96struct nvgpu_clk_arb {
97 spinlock_t sessions_lock;
98 spinlock_t users_lock;
99
100 struct mutex pstate_lock;
101 struct list_head users;
102 struct list_head sessions;
103 struct llist_head requests;
104
105 struct gk20a *g;
106 int status;
107
108 struct nvgpu_clk_arb_target actual_pool[2];
109 struct nvgpu_clk_arb_target *actual;
110
111 u16 gpc2clk_default_mhz;
112 u16 mclk_default_mhz;
113 u32 voltuv_actual;
114
115 struct work_struct update_fn_work;
116 struct workqueue_struct *update_work_queue;
117 struct work_struct vf_table_fn_work;
118 struct workqueue_struct *vf_table_work_queue;
119
120 wait_queue_head_t request_wq;
121
122 struct nvgpu_clk_vf_table *current_vf_table;
123 struct nvgpu_clk_vf_table vf_table_pool[2];
124 u32 vf_table_index;
125
126 u16 *mclk_f_points;
127 atomic_t req_nr;
128
129 u32 mclk_f_numpoints;
130 u16 *gpc2clk_f_points;
131 u32 gpc2clk_f_numpoints;
132
133#ifdef CONFIG_DEBUG_FS
134 struct nvgpu_clk_arb_debug debug_pool[2];
135 struct nvgpu_clk_arb_debug *debug;
136 bool debugfs_set;
137#endif
138};
139
140struct nvgpu_clk_dev {
141 struct nvgpu_clk_session *session;
142 union {
143 struct list_head link;
144 struct llist_node node;
145 };
146 wait_queue_head_t readout_wq;
147 atomic_t poll_mask;
148 u16 gpc2clk_target_mhz;
149 u16 mclk_target_mhz;
150 struct kref refcount;
151};
152
153struct nvgpu_clk_session {
154 bool zombie;
155 struct gk20a *g;
156 struct kref refcount;
157 struct list_head link;
158 struct llist_head targets;
159
160 struct nvgpu_clk_arb_target target_pool[2];
161 struct nvgpu_clk_arb_target *target;
162};
163
164static const struct file_operations completion_dev_ops = {
165 .owner = THIS_MODULE,
166 .release = nvgpu_clk_arb_release_completion_dev,
167 .poll = nvgpu_clk_arb_poll_dev,
168};
169
170static const struct file_operations event_dev_ops = {
171 .owner = THIS_MODULE,
172 .release = nvgpu_clk_arb_release_event_dev,
173 .poll = nvgpu_clk_arb_poll_dev,
174};
175
176int nvgpu_clk_arb_init_arbiter(struct gk20a *g)
177{
178 struct nvgpu_clk_arb *arb;
179 u16 default_mhz;
180 int err;
181 int index;
182 struct nvgpu_clk_vf_table *table;
183
184 gk20a_dbg_fn("");
185
186 if (!g->ops.clk_arb.get_arbiter_clk_domains)
187 return 0;
188
189 arb = kzalloc(sizeof(struct nvgpu_clk_arb), GFP_KERNEL);
190 if (!arb) {
191 err = -ENOMEM;
192 goto init_fail;
193 }
194
195 arb->mclk_f_points = kcalloc(MAX_F_POINTS, sizeof(u16), GFP_KERNEL);
196 if (!arb->mclk_f_points) {
197 err = -ENOMEM;
198 goto init_fail;
199 }
200
201 arb->gpc2clk_f_points = kcalloc(MAX_F_POINTS, sizeof(u16), GFP_KERNEL);
202 if (!arb->gpc2clk_f_points) {
203 err = -ENOMEM;
204 goto init_fail;
205 }
206
207 for (index = 0; index < 2; index++) {
208 table = &arb->vf_table_pool[index];
209 table->gpc2clk_num_points = MAX_F_POINTS;
210 table->mclk_num_points = MAX_F_POINTS;
211
212 table->gpc2clk_points = kcalloc(MAX_F_POINTS,
213 sizeof(struct nvgpu_clk_vf_point), GFP_KERNEL);
214 if (!table->gpc2clk_points) {
215 err = -ENOMEM;
216 goto init_fail;
217 }
218
219
220 table->mclk_points = kcalloc(MAX_F_POINTS,
221 sizeof(struct nvgpu_clk_vf_point), GFP_KERNEL);
222 if (!table->mclk_points) {
223 err = -ENOMEM;
224 goto init_fail;
225 }
226 }
227
228 g->clk_arb = arb;
229 arb->g = g;
230
231 mutex_init(&arb->pstate_lock);
232 spin_lock_init(&arb->sessions_lock);
233 spin_lock_init(&arb->users_lock);
234
235 err = g->ops.clk_arb.get_arbiter_clk_default(g,
236 NVGPU_GPU_CLK_DOMAIN_MCLK, &default_mhz);
237 if (err) {
238 err = -EINVAL;
239 goto init_fail;
240 }
241
242 arb->mclk_default_mhz = default_mhz;
243
244 err = g->ops.clk_arb.get_arbiter_clk_default(g,
245 NVGPU_GPU_CLK_DOMAIN_GPC2CLK, &default_mhz);
246 if (err) {
247 err = -EINVAL;
248 goto init_fail;
249 }
250
251 arb->gpc2clk_default_mhz = default_mhz;
252
253 arb->actual = &arb->actual_pool[0];
254
255 atomic_set(&arb->req_nr, 0);
256
257 INIT_LIST_HEAD_RCU(&arb->users);
258 INIT_LIST_HEAD_RCU(&arb->sessions);
259 init_llist_head(&arb->requests);
260
261 init_waitqueue_head(&arb->request_wq);
262 arb->vf_table_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1,
263 "vf_table_update");
264 arb->update_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1,
265 "arbiter_update");
266
267
268 INIT_WORK(&arb->vf_table_fn_work, nvgpu_clk_arb_run_vf_table_cb);
269
270 INIT_WORK(&arb->update_fn_work, nvgpu_clk_arb_run_arbiter_cb);
271
272#ifdef CONFIG_DEBUG_FS
273 arb->debug = &arb->debug_pool[0];
274
275 if (!arb->debugfs_set) {
276 if (nvgpu_clk_arb_debugfs_init(g))
277 arb->debugfs_set = true;
278 }
279#endif
280 err = clk_vf_point_cache(g);
281 if (err < 0)
282 goto init_fail;
283
284 err = nvgpu_clk_arb_update_vf_table(arb);
285 if (err < 0)
286 goto init_fail;
287 do {
288 /* Check that first run is completed */
289 smp_mb();
290 wait_event_interruptible(arb->request_wq,
291 atomic_read(&arb->req_nr));
292 } while (!atomic_read(&arb->req_nr));
293
294
295 return arb->status;
296
297init_fail:
298
299 kfree(arb->gpc2clk_f_points);
300 kfree(arb->mclk_f_points);
301
302 for (index = 0; index < 2; index++) {
303 kfree(arb->vf_table_pool[index].gpc2clk_points);
304 kfree(arb->vf_table_pool[index].mclk_points);
305 }
306
307 kfree(arb);
308
309 return err;
310}
311
312void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g)
313{
314 kfree(g->clk_arb);
315}
316
317static int nvgpu_clk_arb_install_fd(struct gk20a *g,
318 struct nvgpu_clk_session *session,
319 const struct file_operations *fops,
320 struct nvgpu_clk_dev **_dev)
321{
322 struct file *file;
323 char *name;
324 int fd;
325 int err;
326 struct nvgpu_clk_dev *dev;
327
328 gk20a_dbg_fn("");
329
330 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
331 if (!dev)
332 return -ENOMEM;
333
334 fd = get_unused_fd_flags(O_RDWR);
335 if (fd < 0)
336 return fd;
337
338 name = kasprintf(GFP_KERNEL, "%s-clk-fd%d", dev_name(g->dev), fd);
339 file = anon_inode_getfile(name, fops, dev, O_RDWR);
340 kfree(name);
341 if (IS_ERR(file)) {
342 err = PTR_ERR(file);
343 goto fail;
344 }
345
346 fd_install(fd, file);
347
348 init_waitqueue_head(&dev->readout_wq);
349 atomic_set(&dev->poll_mask, 0);
350
351 dev->session = session;
352 kref_init(&dev->refcount);
353
354 kref_get(&session->refcount);
355
356 *_dev = dev;
357
358 return fd;
359
360fail:
361 kfree(dev);
362 put_unused_fd(fd);
363
364 return err;
365}
366
367int nvgpu_clk_arb_init_session(struct gk20a *g,
368 struct nvgpu_clk_session **_session)
369{
370 struct nvgpu_clk_arb *arb = g->clk_arb;
371 struct nvgpu_clk_session *session = *(_session);
372
373 gk20a_dbg_fn("");
374
375 if (!g->ops.clk_arb.get_arbiter_clk_domains)
376 return 0;
377
378 session = kzalloc(sizeof(struct nvgpu_clk_session), GFP_KERNEL);
379 if (!session)
380 return -ENOMEM;
381 session->g = g;
382
383 kref_init(&session->refcount);
384
385 session->zombie = false;
386 session->target_pool[0].pstate = CTRL_PERF_PSTATE_P8;
387 /* make sure that the initialization of the pool is visible
388 * before the update */
389 smp_wmb();
390 session->target = &session->target_pool[0];
391
392 init_llist_head(&session->targets);
393
394 spin_lock(&arb->sessions_lock);
395 list_add_tail_rcu(&session->link, &arb->sessions);
396 spin_unlock(&arb->sessions_lock);
397
398 *_session = session;
399
400 return 0;
401}
402
403static void nvgpu_clk_arb_free_fd(struct kref *refcount)
404{
405 struct nvgpu_clk_dev *dev = container_of(refcount,
406 struct nvgpu_clk_dev, refcount);
407
408 kfree(dev);
409}
410
411static void nvgpu_clk_arb_free_session(struct kref *refcount)
412{
413 struct nvgpu_clk_session *session = container_of(refcount,
414 struct nvgpu_clk_session, refcount);
415 struct nvgpu_clk_arb *arb = session->g->clk_arb;
416 struct nvgpu_clk_dev *dev, *tmp;
417 struct llist_node *head;
418
419 gk20a_dbg_fn("");
420
421 spin_lock(&arb->sessions_lock);
422 list_del_rcu(&session->link);
423 spin_unlock(&arb->sessions_lock);
424
425 head = llist_del_all(&session->targets);
426 llist_for_each_entry_safe(dev, tmp, head, node) {
427 kref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
428 }
429 synchronize_rcu();
430 kfree(session);
431}
432
433void nvgpu_clk_arb_release_session(struct gk20a *g,
434 struct nvgpu_clk_session *session)
435{
436 struct nvgpu_clk_arb *arb = g->clk_arb;
437
438 gk20a_dbg_fn("");
439
440 session->zombie = true;
441 kref_put(&session->refcount, nvgpu_clk_arb_free_session);
442
443 queue_work(arb->update_work_queue, &arb->update_fn_work);
444}
445
446int nvgpu_clk_arb_install_event_fd(struct gk20a *g,
447 struct nvgpu_clk_session *session, int *event_fd)
448{
449 struct nvgpu_clk_arb *arb = g->clk_arb;
450 struct nvgpu_clk_dev *dev;
451 int fd;
452
453 gk20a_dbg_fn("");
454
455 fd = nvgpu_clk_arb_install_fd(g, session, &event_dev_ops, &dev);
456 if (fd < 0)
457 return fd;
458
459 spin_lock(&arb->users_lock);
460 list_add_tail_rcu(&dev->link, &arb->users);
461 spin_unlock(&arb->users_lock);
462
463 *event_fd = fd;
464
465 return 0;
466}
467
468int nvgpu_clk_arb_install_request_fd(struct gk20a *g,
469 struct nvgpu_clk_session *session, int *request_fd)
470{
471 struct nvgpu_clk_dev *dev;
472 int fd;
473
474 gk20a_dbg_fn("");
475
476 fd = nvgpu_clk_arb_install_fd(g, session, &completion_dev_ops, &dev);
477 if (fd < 0)
478 return fd;
479
480 *request_fd = fd;
481
482 return 0;
483}
484
485static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb)
486{
487 struct gk20a *g = arb->g;
488 struct nvgpu_clk_vf_table *table;
489
490 u32 i, j;
491 int status = -EINVAL;
492 u32 gpc2clk_voltuv = 0, mclk_voltuv = 0;
493 u32 gpc2clk_voltuv_sram = 0, mclk_voltuv_sram = 0;
494 u16 gpc2clk_min, gpc2clk_max, clk_cur;
495 u16 mclk_min, mclk_max;
496 u32 num_points;
497
498 struct clk_set_info *p5_info, *p0_info;
499
500
501 table = ACCESS_ONCE(arb->current_vf_table);
502 /* make flag visible when all data has resolved in the tables */
503 smp_rmb();
504
505 table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] :
506 &arb->vf_table_pool[0];
507
508 /* Get allowed memory ranges */
509 if (nvgpu_clk_arb_get_arbiter_clk_range(g, NVGPU_GPU_CLK_DOMAIN_GPC2CLK,
510 &gpc2clk_min,
511 &gpc2clk_max) < 0) {
512 gk20a_err(dev_from_gk20a(g),
513 "failed to fetch GPC2CLK range");
514 goto exit_vf_table;
515 }
516 if (nvgpu_clk_arb_get_arbiter_clk_range(g, NVGPU_GPU_CLK_DOMAIN_MCLK,
517 &mclk_min, &mclk_max) < 0) {
518 gk20a_err(dev_from_gk20a(g),
519 "failed to fetch MCLK range");
520 goto exit_vf_table;
521 }
522
523 if (clk_domain_get_f_points(arb->g, NVGPU_GPU_CLK_DOMAIN_GPC2CLK,
524 &table->gpc2clk_num_points, arb->gpc2clk_f_points)) {
525 gk20a_err(dev_from_gk20a(g),
526 "failed to fetch GPC2CLK frequency points");
527 goto exit_vf_table;
528 }
529
530 table->gpc2clk_num_points = MAX_F_POINTS;
531 table->mclk_num_points = MAX_F_POINTS;
532
533 if (clk_domain_get_f_points(arb->g, NVGPU_GPU_CLK_DOMAIN_MCLK,
534 &table->mclk_num_points, arb->mclk_f_points)) {
535 gk20a_err(dev_from_gk20a(g),
536 "failed to fetch MCLK frequency points");
537 goto exit_vf_table;
538 }
539 if (!table->mclk_num_points || !table->gpc2clk_num_points) {
540 gk20a_err(dev_from_gk20a(g),
541 "empty queries to f points mclk %d gpc2clk %d",
542 table->mclk_num_points, table->gpc2clk_num_points);
543 status = -EINVAL;
544 goto exit_vf_table;
545 }
546
547 memset(table->mclk_points, 0,
548 table->mclk_num_points*sizeof(struct nvgpu_clk_vf_point));
549 memset(table->gpc2clk_points, 0,
550 table->gpc2clk_num_points*sizeof(struct nvgpu_clk_vf_point));
551
552 p5_info = pstate_get_clk_set_info(g,
553 CTRL_PERF_PSTATE_P5, clkwhich_mclk);
554 if (!p5_info) {
555 gk20a_err(dev_from_gk20a(g),
556 "failed to get MCLK P5 info");
557 goto exit_vf_table;
558 }
559 p0_info = pstate_get_clk_set_info(g,
560 CTRL_PERF_PSTATE_P0, clkwhich_mclk);
561 if (!p0_info) {
562 gk20a_err(dev_from_gk20a(g),
563 "failed to get MCLK P0 info");
564 goto exit_vf_table;
565 }
566
567 for (i = 0, j = 0, num_points = 0, clk_cur = 0;
568 i < table->mclk_num_points; i++) {
569
570 if ((arb->mclk_f_points[i] >= mclk_min) &&
571 (arb->mclk_f_points[i] <= mclk_max) &&
572 (arb->mclk_f_points[i] != clk_cur)) {
573
574 table->mclk_points[j].mem_mhz = arb->mclk_f_points[i];
575 mclk_voltuv = mclk_voltuv_sram = 0;
576
577 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK,
578 &table->mclk_points[j].mem_mhz, &mclk_voltuv,
579 CTRL_VOLT_DOMAIN_LOGIC);
580 if (status < 0) {
581 gk20a_err(dev_from_gk20a(g),
582 "failed to get MCLK LOGIC voltage");
583 goto exit_vf_table;
584 }
585 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK,
586 &table->mclk_points[j].mem_mhz,
587 &mclk_voltuv_sram,
588 CTRL_VOLT_DOMAIN_SRAM);
589 if (status < 0) {
590 gk20a_err(dev_from_gk20a(g),
591 "failed to get MCLK SRAM voltage");
592 goto exit_vf_table;
593 }
594
595 table->mclk_points[j].uvolt = mclk_voltuv;
596 table->mclk_points[j].uvolt_sram = mclk_voltuv_sram;
597 clk_cur = table->mclk_points[j].mem_mhz;
598
599 if ((clk_cur >= p5_info->min_mhz) &&
600 (clk_cur <= p5_info->max_mhz))
601 VF_POINT_SET_PSTATE_SUPPORTED(
602 &table->mclk_points[j],
603 CTRL_PERF_PSTATE_P5);
604 if ((clk_cur >= p0_info->min_mhz) &&
605 (clk_cur <= p0_info->max_mhz))
606 VF_POINT_SET_PSTATE_SUPPORTED(
607 &table->mclk_points[j],
608 CTRL_PERF_PSTATE_P0);
609
610 j++;
611 num_points++;
612
613 }
614 }
615 table->mclk_num_points = num_points;
616
617 p5_info = pstate_get_clk_set_info(g,
618 CTRL_PERF_PSTATE_P5, clkwhich_gpc2clk);
619 if (!p5_info) {
620 status = -EINVAL;
621 gk20a_err(dev_from_gk20a(g),
622 "failed to get GPC2CLK P5 info");
623 goto exit_vf_table;
624 }
625
626 p0_info = pstate_get_clk_set_info(g,
627 CTRL_PERF_PSTATE_P0, clkwhich_gpc2clk);
628 if (!p0_info) {
629 status = -EINVAL;
630 gk20a_err(dev_from_gk20a(g),
631 "failed to get GPC2CLK P0 info");
632 goto exit_vf_table;
633 }
634
635 /* GPC2CLK needs to be checked in two passes. The first determines the
636 * relationships between GPC2CLK, SYS2CLK and XBAR2CLK, while the
637 * second verifies that the clocks minimum is satisfied and sets
638 * the voltages
639 */
640 for (i = 0, j = 0, num_points = 0, clk_cur = 0;
641 i < table->gpc2clk_num_points; i++) {
642 struct set_fll_clk setfllclk;
643
644 if ((arb->gpc2clk_f_points[i] >= gpc2clk_min) &&
645 (arb->gpc2clk_f_points[i] <= gpc2clk_max) &&
646 (arb->gpc2clk_f_points[i] != clk_cur)) {
647
648 table->gpc2clk_points[j].gpc_mhz =
649 arb->gpc2clk_f_points[i];
650 setfllclk.gpc2clkmhz = arb->gpc2clk_f_points[i];
651 status = clk_get_fll_clks(g, &setfllclk);
652 if (status < 0) {
653 gk20a_err(dev_from_gk20a(g),
654 "failed to get GPC2CLK slave clocks");
655 goto exit_vf_table;
656 }
657
658 table->gpc2clk_points[j].sys_mhz =
659 setfllclk.sys2clkmhz;
660 table->gpc2clk_points[j].xbar_mhz =
661 setfllclk.xbar2clkmhz;
662
663 clk_cur = table->gpc2clk_points[j].gpc_mhz;
664
665 if ((clk_cur >= p5_info->min_mhz) &&
666 (clk_cur <= p5_info->max_mhz))
667 VF_POINT_SET_PSTATE_SUPPORTED(
668 &table->gpc2clk_points[j],
669 CTRL_PERF_PSTATE_P5);
670 if ((clk_cur >= p0_info->min_mhz) &&
671 (clk_cur <= p0_info->max_mhz))
672 VF_POINT_SET_PSTATE_SUPPORTED(
673 &table->gpc2clk_points[j],
674 CTRL_PERF_PSTATE_P0);
675
676 j++;
677 num_points++;
678 }
679 }
680 table->gpc2clk_num_points = num_points;
681
682 /* Second pass */
683 for (i = 0, j = 0; i < table->gpc2clk_num_points; i++) {
684
685 u16 alt_gpc2clk = table->gpc2clk_points[i].gpc_mhz;
686 gpc2clk_voltuv = gpc2clk_voltuv_sram = 0;
687
688 /* Check sysclk */
689 p5_info = pstate_get_clk_set_info(g,
690 VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]),
691 clkwhich_sys2clk);
692 if (!p5_info) {
693 status = -EINVAL;
694 gk20a_err(dev_from_gk20a(g),
695 "failed to get SYS2CLK P5 info");
696 goto exit_vf_table;
697 }
698
699 /* sys2clk below clk min, need to find correct clock */
700 if (table->gpc2clk_points[i].sys_mhz < p5_info->min_mhz) {
701 for (j = i + 1; j < table->gpc2clk_num_points; j++) {
702
703 if (table->gpc2clk_points[j].sys_mhz >=
704 p5_info->min_mhz) {
705
706
707 table->gpc2clk_points[i].sys_mhz =
708 p5_info->min_mhz;
709
710 alt_gpc2clk = alt_gpc2clk <
711 table->gpc2clk_points[j].
712 gpc_mhz ?
713 table->gpc2clk_points[j].
714 gpc_mhz:
715 alt_gpc2clk;
716 break;
717 }
718 }
719 /* no VF exists that satisfies condition */
720 if (j == table->gpc2clk_num_points) {
721 gk20a_err(dev_from_gk20a(g),
722 "NO SYS2CLK VF point possible");
723 status = -EINVAL;
724 goto exit_vf_table;
725 }
726 }
727
728 /* Check xbarclk */
729 p5_info = pstate_get_clk_set_info(g,
730 VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]),
731 clkwhich_xbar2clk);
732 if (!p5_info) {
733 status = -EINVAL;
734 gk20a_err(dev_from_gk20a(g),
735 "failed to get SYS2CLK P5 info");
736 goto exit_vf_table;
737 }
738
739 /* xbar2clk below clk min, need to find correct clock */
740 if (table->gpc2clk_points[i].xbar_mhz < p5_info->min_mhz) {
741 for (j = i; j < table->gpc2clk_num_points; j++) {
742 if (table->gpc2clk_points[j].xbar_mhz >=
743 p5_info->min_mhz) {
744
745 table->gpc2clk_points[i].xbar_mhz =
746 p5_info->min_mhz;
747
748 alt_gpc2clk = alt_gpc2clk <
749 table->gpc2clk_points[j].
750 gpc_mhz ?
751 table->gpc2clk_points[j].
752 gpc_mhz:
753 alt_gpc2clk;
754 break;
755 }
756 }
757 /* no VF exists that satisfies condition */
758 if (j == table->gpc2clk_num_points) {
759 status = -EINVAL;
760 gk20a_err(dev_from_gk20a(g),
761 "NO XBAR2CLK VF point possible");
762
763 goto exit_vf_table;
764 }
765 }
766
767 /* Calculate voltages */
768 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK,
769 &alt_gpc2clk, &gpc2clk_voltuv,
770 CTRL_VOLT_DOMAIN_LOGIC);
771 if (status < 0) {
772 gk20a_err(dev_from_gk20a(g),
773 "failed to get GPC2CLK LOGIC voltage");
774 goto exit_vf_table;
775 }
776
777 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK,
778 &alt_gpc2clk,
779 &gpc2clk_voltuv_sram,
780 CTRL_VOLT_DOMAIN_SRAM);
781 if (status < 0) {
782 gk20a_err(dev_from_gk20a(g),
783 "failed to get GPC2CLK SRAM voltage");
784 goto exit_vf_table;
785 }
786
787 table->gpc2clk_points[i].uvolt = gpc2clk_voltuv;
788 table->gpc2clk_points[i].uvolt_sram = gpc2clk_voltuv_sram;
789 }
790
791 /* make table visible when all data has resolved in the tables */
792 smp_wmb();
793 xchg(&arb->current_vf_table, table);
794
795 queue_work(arb->update_work_queue, &arb->update_fn_work);
796exit_vf_table:
797
798 return status;
799}
800
801void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g)
802{
803 struct nvgpu_clk_arb *arb = g->clk_arb;
804
805 queue_work(arb->vf_table_work_queue, &arb->vf_table_fn_work);
806}
807
808static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work)
809{
810 struct nvgpu_clk_arb *arb =
811 container_of(work, struct nvgpu_clk_arb, vf_table_fn_work);
812 struct gk20a *g = arb->g;
813 u32 err;
814
815 /* get latest vf curve from pmu */
816 err = clk_vf_point_cache(g);
817 if (err) {
818 gk20a_err(dev_from_gk20a(g),
819 "failed to cache VF table");
820 return;
821 }
822 nvgpu_clk_arb_update_vf_table(arb);
823}
824
825static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work)
826{
827 struct nvgpu_clk_arb *arb =
828 container_of(work, struct nvgpu_clk_arb, update_fn_work);
829 struct nvgpu_clk_session *session;
830 struct nvgpu_clk_dev *dev;
831 struct nvgpu_clk_dev *tmp;
832 struct nvgpu_clk_arb_target *target, *actual;
833 struct gk20a *g = arb->g;
834 struct llist_node *head;
835
836 u32 pstate = VF_POINT_INVALID_PSTATE;
837 u32 voltuv, voltuv_sram;
838 bool mclk_set, gpc2clk_set;
839 u32 nuvmin, nuvmin_sram;
840
841 int status = 0;
842
843 /* Temporary variables for checking target frequency */
844 u16 gpc2clk_target, sys2clk_target, xbar2clk_target, mclk_target;
845
846#ifdef CONFIG_DEBUG_FS
847 u64 t0, t1;
848 struct nvgpu_clk_arb_debug *debug;
849
850#endif
851
852 gk20a_dbg_fn("");
853
854#ifdef CONFIG_DEBUG_FS
855 g->ops.read_ptimer(g, &t0);
856#endif
857
858 /* Only one arbiter should be running */
859 gpc2clk_target = 0;
860 mclk_target = 0;
861
862 rcu_read_lock();
863 list_for_each_entry_rcu(session, &arb->sessions, link) {
864 if (!session->zombie) {
865 mclk_set = false;
866 gpc2clk_set = false;
867 target = ACCESS_ONCE(session->target) ==
868 &session->target_pool[0] ?
869 &session->target_pool[1] :
870 &session->target_pool[0];
871 /* Do not reorder pointer */
872 smp_rmb();
873 head = llist_del_all(&session->targets);
874 if (head) {
875
876 /* Copy over state */
877 target->mclk = session->target->mclk;
878 target->gpc2clk = session->target->gpc2clk;
879 /* Query the latest committed request */
880 llist_for_each_entry_safe(dev, tmp, head,
881 node) {
882 if (!mclk_set && dev->mclk_target_mhz) {
883 target->mclk =
884 dev->mclk_target_mhz;
885 mclk_set = true;
886 }
887 if (!gpc2clk_set &&
888 dev->gpc2clk_target_mhz) {
889 target->gpc2clk =
890 dev->gpc2clk_target_mhz;
891 gpc2clk_set = true;
892 }
893 kref_get(&dev->refcount);
894 llist_add(&dev->node, &arb->requests);
895 }
896 /* Ensure target is updated before ptr sawp */
897 smp_wmb();
898 xchg(&session->target, target);
899 }
900
901 mclk_target = mclk_target > session->target->mclk ?
902 mclk_target : session->target->mclk;
903
904 gpc2clk_target =
905 gpc2clk_target > session->target->gpc2clk ?
906 gpc2clk_target : session->target->gpc2clk;
907 }
908 }
909 rcu_read_unlock();
910
911 gpc2clk_target = (gpc2clk_target > 0) ? gpc2clk_target :
912 arb->gpc2clk_default_mhz;
913
914 mclk_target = (mclk_target > 0) ? mclk_target:
915 arb->mclk_default_mhz;
916
917 sys2clk_target = 0;
918 xbar2clk_target = 0;
919 /* Query the table for the closest vf point to program */
920 pstate = nvgpu_clk_arb_find_vf_point(arb, &gpc2clk_target,
921 &sys2clk_target, &xbar2clk_target, &mclk_target, &voltuv,
922 &voltuv_sram, &nuvmin, &nuvmin_sram);
923
924 if (pstate == VF_POINT_INVALID_PSTATE) {
925 arb->status = -EINVAL;
926 /* make status visible */
927 smp_mb();
928 goto exit_arb;
929 }
930
931 if ((arb->actual->gpc2clk == gpc2clk_target) &&
932 (arb->actual->mclk == mclk_target) &&
933 (arb->voltuv_actual == voltuv)) {
934 goto exit_arb;
935 }
936
937 /* Program clocks */
938 /* A change in both mclk of gpc2clk may require a change in voltage */
939
940 mutex_lock(&arb->pstate_lock);
941 status = nvgpu_lpwr_disable_pg(g, false);
942
943 status = clk_pmu_freq_controller_load(g, false);
944 if (status < 0) {
945 arb->status = status;
946 mutex_unlock(&arb->pstate_lock);
947
948 /* make status visible */
949 smp_mb();
950 goto exit_arb;
951 }
952 status = volt_set_noiseaware_vmin(g, nuvmin, nuvmin_sram);
953 if (status < 0) {
954 arb->status = status;
955 mutex_unlock(&arb->pstate_lock);
956
957 /* make status visible */
958 smp_mb();
959 goto exit_arb;
960 }
961
962 status = nvgpu_clk_arb_change_vf_point(g, gpc2clk_target,
963 sys2clk_target, xbar2clk_target, mclk_target, voltuv,
964 voltuv_sram);
965 if (status < 0) {
966 arb->status = status;
967 mutex_unlock(&arb->pstate_lock);
968
969 /* make status visible */
970 smp_mb();
971 goto exit_arb;
972 }
973
974 status = clk_pmu_freq_controller_load(g, true);
975 if (status < 0) {
976 arb->status = status;
977 mutex_unlock(&arb->pstate_lock);
978
979 /* make status visible */
980 smp_mb();
981 goto exit_arb;
982 }
983
984 status = nvgpu_lwpr_mclk_change(g, pstate);
985 if (status < 0) {
986 arb->status = status;
987 mutex_unlock(&arb->pstate_lock);
988
989 /* make status visible */
990 smp_mb();
991 goto exit_arb;
992 }
993
994 actual = ACCESS_ONCE(arb->actual) == &arb->actual_pool[0] ?
995 &arb->actual_pool[1] : &arb->actual_pool[0];
996
997 /* do not reorder this pointer */
998 smp_rmb();
999 actual->gpc2clk = gpc2clk_target;
1000 actual->mclk = mclk_target;
1001 arb->voltuv_actual = voltuv;
1002 actual->pstate = pstate;
1003 arb->status = status;
1004
1005 /* Make changes visible to other threads */
1006 smp_wmb();
1007 xchg(&arb->actual, actual);
1008
1009 status = nvgpu_lpwr_enable_pg(g, false);
1010 if (status < 0) {
1011 arb->status = status;
1012 mutex_unlock(&arb->pstate_lock);
1013
1014 /* make status visible */
1015 smp_mb();
1016 goto exit_arb;
1017 }
1018
1019 /* status must be visible before atomic inc */
1020 smp_wmb();
1021 atomic_inc(&arb->req_nr);
1022
1023 /* Unlock pstate change for PG */
1024 mutex_unlock(&arb->pstate_lock);
1025
1026 wake_up_interruptible(&arb->request_wq);
1027
1028#ifdef CONFIG_DEBUG_FS
1029 g->ops.read_ptimer(g, &t1);
1030
1031 debug = arb->debug == &arb->debug_pool[0] ?
1032 &arb->debug_pool[1] : &arb->debug_pool[0];
1033
1034 memcpy(debug, arb->debug, sizeof(arb->debug_pool[0]));
1035 debug->switch_num++;
1036
1037 if (debug->switch_num == 1) {
1038 debug->switch_max = debug->switch_min =
1039 debug->switch_avg = (t1-t0)/1000;
1040 debug->switch_std = 0;
1041 } else {
1042 s64 prev_avg;
1043 s64 curr = (t1-t0)/1000;
1044
1045 debug->switch_max = curr > debug->switch_max ?
1046 curr : debug->switch_max;
1047 debug->switch_min = debug->switch_min ?
1048 (curr < debug->switch_min ?
1049 curr : debug->switch_min) : curr;
1050 prev_avg = debug->switch_avg;
1051 debug->switch_avg = (curr +
1052 (debug->switch_avg * (debug->switch_num-1))) /
1053 debug->switch_num;
1054 debug->switch_std +=
1055 (curr - debug->switch_avg) * (curr - prev_avg);
1056 }
1057 /* commit changes before exchanging debug pointer */
1058 smp_wmb();
1059 xchg(&arb->debug, debug);
1060#endif
1061
1062exit_arb:
1063 if (status < 0)
1064 gk20a_err(dev_from_gk20a(g),
1065 "Error in arbiter update");
1066
1067 /* notify completion for all requests */
1068 head = llist_del_all(&arb->requests);
1069 llist_for_each_entry_safe(dev, tmp, head, node) {
1070 atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM);
1071 wake_up_interruptible(&dev->readout_wq);
1072 kref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1073 }
1074
1075 /* notify event for all users */
1076 rcu_read_lock();
1077 list_for_each_entry_rcu(dev, &arb->users, link) {
1078 atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM);
1079 wake_up_interruptible(&dev->readout_wq);
1080 }
1081 rcu_read_unlock();
1082}
1083
1084int nvgpu_clk_arb_commit_request_fd(struct gk20a *g,
1085 struct nvgpu_clk_session *session, int request_fd)
1086{
1087 struct nvgpu_clk_arb *arb = g->clk_arb;
1088 struct nvgpu_clk_dev *dev;
1089 struct fd fd;
1090 int err = 0;
1091
1092 gk20a_dbg_fn("");
1093
1094 fd = fdget(request_fd);
1095
1096 if (!fd.file)
1097 return -EINVAL;
1098
1099 dev = (struct nvgpu_clk_dev *) fd.file->private_data;
1100
1101 if (!dev || dev->session != session) {
1102 err = -EINVAL;
1103 goto fdput_fd;
1104 }
1105 kref_get(&dev->refcount);
1106 llist_add(&dev->node, &session->targets);
1107
1108 queue_work(arb->update_work_queue, &arb->update_fn_work);
1109
1110fdput_fd:
1111 fdput(fd);
1112 return err;
1113}
1114
1115static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait)
1116{
1117 struct nvgpu_clk_dev *dev = filp->private_data;
1118
1119 gk20a_dbg_fn("");
1120
1121 poll_wait(filp, &dev->readout_wq, wait);
1122 return atomic_xchg(&dev->poll_mask, 0);
1123}
1124
1125static int nvgpu_clk_arb_release_completion_dev(struct inode *inode,
1126 struct file *filp)
1127{
1128 struct nvgpu_clk_dev *dev = filp->private_data;
1129 struct nvgpu_clk_session *session = dev->session;
1130 struct nvgpu_clk_arb *arb;
1131
1132 arb = session->g->clk_arb;
1133
1134 gk20a_dbg_fn("");
1135
1136 kref_put(&session->refcount, nvgpu_clk_arb_free_session);
1137 kref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1138
1139 return 0;
1140}
1141
1142static int nvgpu_clk_arb_release_event_dev(struct inode *inode,
1143 struct file *filp)
1144{
1145 struct nvgpu_clk_dev *dev = filp->private_data;
1146 struct nvgpu_clk_session *session = dev->session;
1147 struct nvgpu_clk_arb *arb;
1148
1149 arb = session->g->clk_arb;
1150
1151 gk20a_dbg_fn("");
1152
1153 spin_lock(&arb->users_lock);
1154 list_del_rcu(&dev->link);
1155 spin_unlock(&arb->users_lock);
1156
1157 kref_put(&session->refcount, nvgpu_clk_arb_free_session);
1158 synchronize_rcu();
1159 kfree(dev);
1160
1161 return 0;
1162}
1163
1164int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session,
1165 int request_fd, u32 api_domain, u16 target_mhz)
1166{
1167 struct nvgpu_clk_dev *dev;
1168 struct fd fd;
1169 int err = 0;
1170
1171 gk20a_dbg_fn("domain=0x%08x target_mhz=%u", api_domain, target_mhz);
1172
1173 fd = fdget(request_fd);
1174
1175 if (!fd.file)
1176 return -EINVAL;
1177
1178 dev = fd.file->private_data;
1179 if (!dev || dev->session != session) {
1180 err = -EINVAL;
1181 goto fdput_fd;
1182 }
1183
1184 switch (api_domain) {
1185 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1186 dev->mclk_target_mhz = target_mhz;
1187 break;
1188
1189 case NVGPU_GPU_CLK_DOMAIN_GPC2CLK:
1190 dev->gpc2clk_target_mhz = target_mhz;
1191 break;
1192
1193 default:
1194 err = -EINVAL;
1195 }
1196
1197fdput_fd:
1198 fdput(fd);
1199 return err;
1200}
1201
1202int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session,
1203 u32 api_domain, u16 *freq_mhz)
1204{
1205 int err = 0;
1206 struct nvgpu_clk_arb_target *target;
1207
1208 do {
1209 target = ACCESS_ONCE(session->target);
1210 /* no reordering of this pointer */
1211 smp_rmb();
1212
1213 switch (api_domain) {
1214 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1215 *freq_mhz = target->mclk;
1216 break;
1217
1218 case NVGPU_GPU_CLK_DOMAIN_GPC2CLK:
1219 *freq_mhz = target->gpc2clk;
1220 break;
1221
1222 default:
1223 *freq_mhz = 0;
1224 err = -EINVAL;
1225 }
1226 } while (target != ACCESS_ONCE(session->target));
1227 return err;
1228}
1229
1230int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g,
1231 u32 api_domain, u16 *freq_mhz)
1232{
1233 struct nvgpu_clk_arb *arb = g->clk_arb;
1234 int err = 0;
1235 struct nvgpu_clk_arb_target *actual;
1236
1237 do {
1238 actual = ACCESS_ONCE(arb->actual);
1239 /* no reordering of this pointer */
1240 smp_rmb();
1241
1242 switch (api_domain) {
1243 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1244 *freq_mhz = actual->mclk;
1245 break;
1246
1247 case NVGPU_GPU_CLK_DOMAIN_GPC2CLK:
1248 *freq_mhz = actual->gpc2clk;
1249 break;
1250
1251 default:
1252 *freq_mhz = 0;
1253 err = -EINVAL;
1254 }
1255 } while (actual != ACCESS_ONCE(arb->actual));
1256 return err;
1257}
1258
1259int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g,
1260 u32 api_domain, u16 *freq_mhz)
1261{
1262
1263 *freq_mhz = g->ops.clk.get_rate(g, api_domain);
1264 return 0;
1265}
1266
1267int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
1268 u16 *min_mhz, u16 *max_mhz)
1269{
1270 return g->ops.clk_arb.get_arbiter_clk_range(g, api_domain,
1271 min_mhz, max_mhz);
1272}
1273
1274u32 nvgpu_clk_arb_get_arbiter_clk_domains(struct gk20a *g)
1275{
1276 return g->ops.clk_arb.get_arbiter_clk_domains(g);
1277}
1278
1279int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g,
1280 u32 api_domain, u32 *max_points, u16 *fpoints)
1281{
1282 return (int)clk_domain_get_f_points(g, api_domain, max_points, fpoints);
1283}
1284
1285static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb,
1286 u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk,
1287 u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram)
1288{
1289 u16 gpc2clk_target, mclk_target;
1290 u32 gpc2clk_voltuv, gpc2clk_voltuv_sram;
1291 u32 mclk_voltuv, mclk_voltuv_sram;
1292 u32 pstate = VF_POINT_INVALID_PSTATE;
1293 struct nvgpu_clk_vf_table *table;
1294 u32 index, index_mclk;
1295 struct nvgpu_clk_vf_point *mclk_vf = NULL;
1296
1297 do {
1298 gpc2clk_target = *gpc2clk;
1299 mclk_target = *mclk;
1300 gpc2clk_voltuv = 0;
1301 gpc2clk_voltuv_sram = 0;
1302 mclk_voltuv = 0;
1303 mclk_voltuv_sram = 0;
1304
1305 table = ACCESS_ONCE(arb->current_vf_table);
1306 /* pointer to table can be updated by callback */
1307 smp_rmb();
1308
1309 if (!table)
1310 continue;
1311 if ((!table->gpc2clk_num_points) || (!table->mclk_num_points)) {
1312 gk20a_err(dev_from_gk20a(arb->g), "found empty table");
1313 goto find_exit;
1314 }
1315 /* First we check MCLK to find out which PSTATE we are
1316 * are requesting, and from there try to find the minimum
1317 * GPC2CLK on the same PSTATE that satisfies the request.
1318 * If no GPC2CLK can be found, then we need to up the PSTATE
1319 */
1320
1321recalculate_vf_point:
1322 for (index = 0; index < table->mclk_num_points; index++) {
1323 if (table->mclk_points[index].mem_mhz >= mclk_target) {
1324 mclk_vf = &table->mclk_points[index];
1325 break;
1326 }
1327 }
1328 if (index == table->mclk_num_points) {
1329 mclk_vf = &table->mclk_points[index-1];
1330 index = table->mclk_num_points - 1;
1331 }
1332 index_mclk = index;
1333
1334 /* round up the freq requests */
1335 for (index = 0; index < table->gpc2clk_num_points; index++) {
1336 pstate = VF_POINT_COMMON_PSTATE(
1337 &table->gpc2clk_points[index], mclk_vf);
1338
1339 if ((table->gpc2clk_points[index].gpc_mhz >=
1340 gpc2clk_target) &&
1341 (pstate != VF_POINT_INVALID_PSTATE)){
1342 gpc2clk_target =
1343 table->gpc2clk_points[index].gpc_mhz;
1344 *sys2clk =
1345 table->gpc2clk_points[index].sys_mhz;
1346 *xbar2clk =
1347 table->gpc2clk_points[index].xbar_mhz;
1348
1349 gpc2clk_voltuv =
1350 table->gpc2clk_points[index].uvolt;
1351 gpc2clk_voltuv_sram =
1352 table->gpc2clk_points[index].uvolt_sram;
1353 break;
1354 }
1355 }
1356
1357 if (index == table->gpc2clk_num_points) {
1358 pstate = VF_POINT_COMMON_PSTATE(
1359 &table->gpc2clk_points[index-1], mclk_vf);
1360 if (pstate != VF_POINT_INVALID_PSTATE) {
1361 gpc2clk_target =
1362 table->gpc2clk_points[index-1].gpc_mhz;
1363 *sys2clk =
1364 table->gpc2clk_points[index-1].sys_mhz;
1365 *xbar2clk =
1366 table->gpc2clk_points[index-1].xbar_mhz;
1367
1368 gpc2clk_voltuv =
1369 table->gpc2clk_points[index-1].uvolt;
1370 gpc2clk_voltuv_sram =
1371 table->gpc2clk_points[index-1].
1372 uvolt_sram;
1373 } else if (index_mclk >= table->mclk_num_points - 1) {
1374 /* There is no available combination of MCLK
1375 * and GPC2CLK, we need to fail this
1376 */
1377 gpc2clk_target = 0;
1378 mclk_target = 0;
1379 pstate = VF_POINT_INVALID_PSTATE;
1380 goto find_exit;
1381 } else {
1382 /* recalculate with higher PSTATE */
1383 gpc2clk_target = *gpc2clk;
1384 mclk_target = table->mclk_points[index_mclk+1].
1385 mem_mhz;
1386 goto recalculate_vf_point;
1387 }
1388 }
1389
1390 mclk_target = mclk_vf->mem_mhz;
1391 mclk_voltuv = mclk_vf->uvolt;
1392 mclk_voltuv_sram = mclk_vf->uvolt_sram;
1393
1394 } while (!table ||
1395 (ACCESS_ONCE(arb->current_vf_table) != table));
1396
1397find_exit:
1398 *voltuv = gpc2clk_voltuv > mclk_voltuv ? gpc2clk_voltuv : mclk_voltuv;
1399 *voltuv_sram = gpc2clk_voltuv_sram > mclk_voltuv_sram ?
1400 gpc2clk_voltuv_sram : mclk_voltuv_sram;
1401 /* noise unaware vmin */
1402 *nuvmin = mclk_voltuv;
1403 *nuvmin_sram = mclk_voltuv_sram;
1404 *gpc2clk = gpc2clk_target;
1405 *mclk = mclk_target;
1406 return pstate;
1407}
1408
1409/* This function is inherently unsafe to call while arbiter is running
1410 * arbiter must be blocked before calling this function */
1411int nvgpu_clk_arb_get_current_pstate(struct gk20a *g)
1412{
1413 return ACCESS_ONCE(g->clk_arb->actual->pstate);
1414}
1415
1416static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target,
1417 u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv,
1418 u32 voltuv_sram)
1419{
1420 struct set_fll_clk fllclk;
1421 struct nvgpu_clk_arb *arb = g->clk_arb;
1422 int status;
1423
1424 fllclk.gpc2clkmhz = gpc2clk_target;
1425 fllclk.sys2clkmhz = sys2clk_target;
1426 fllclk.xbar2clkmhz = xbar2clk_target;
1427
1428 fllclk.voltuv = voltuv;
1429
1430 /* if voltage ascends we do:
1431 * (1) FLL change
1432 * (2) Voltage change
1433 * (3) MCLK change
1434 * If it goes down
1435 * (1) MCLK change
1436 * (2) Voltage change
1437 * (3) FLL change
1438 */
1439
1440 /* descending */
1441 if (voltuv < arb->voltuv_actual) {
1442 status = g->clk_pmu.clk_mclk.change(g, mclk_target);
1443 if (status < 0)
1444 return status;
1445
1446 status = volt_set_voltage(g, voltuv, voltuv_sram);
1447 if (status < 0)
1448 return status;
1449
1450 status = clk_set_fll_clks(g, &fllclk);
1451 if (status < 0)
1452 return status;
1453 } else {
1454 status = clk_set_fll_clks(g, &fllclk);
1455 if (status < 0)
1456 return status;
1457
1458 status = volt_set_voltage(g, voltuv, voltuv_sram);
1459 if (status < 0)
1460 return status;
1461
1462 status = g->clk_pmu.clk_mclk.change(g, mclk_target);
1463 if (status < 0)
1464 return status;
1465 }
1466
1467 return 0;
1468}
1469
1470void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock)
1471{
1472 struct nvgpu_clk_arb *arb = g->clk_arb;
1473
1474 if (lock)
1475 mutex_lock(&arb->pstate_lock);
1476 else
1477 mutex_unlock(&arb->pstate_lock);
1478
1479}
1480
1481#ifdef CONFIG_DEBUG_FS
1482static int nvgpu_clk_arb_stats_show(struct seq_file *s, void *unused)
1483{
1484 struct gk20a *g = s->private;
1485 struct nvgpu_clk_arb *arb = g->clk_arb;
1486 struct nvgpu_clk_arb_debug *debug;
1487
1488 u64 num;
1489 s64 tmp, avg, std, max, min;
1490
1491 debug = ACCESS_ONCE(arb->debug);
1492 /* Make copy of structure and ensure no reordering */
1493 smp_rmb();
1494 if (!debug)
1495 return -EINVAL;
1496
1497 std = debug->switch_std;
1498 avg = debug->switch_avg;
1499 max = debug->switch_max;
1500 min = debug->switch_min;
1501 num = debug->switch_num;
1502
1503 tmp = std;
1504 do_div(tmp, num);
1505 seq_printf(s, "Number of transitions: %lld\n",
1506 num);
1507 seq_printf(s, "max / min : %lld / %lld usec\n",
1508 max, min);
1509 seq_printf(s, "avg / std : %lld / %ld usec\n",
1510 avg, int_sqrt(tmp));
1511
1512 return 0;
1513}
1514
1515static int nvgpu_clk_arb_stats_open(struct inode *inode, struct file *file)
1516{
1517 return single_open(file, nvgpu_clk_arb_stats_show, inode->i_private);
1518}
1519
1520static const struct file_operations nvgpu_clk_arb_stats_fops = {
1521 .open = nvgpu_clk_arb_stats_open,
1522 .read = seq_read,
1523 .llseek = seq_lseek,
1524 .release = single_release,
1525};
1526
1527
1528static int nvgpu_clk_arb_debugfs_init(struct gk20a *g)
1529{
1530 struct gk20a_platform *platform = dev_get_drvdata(g->dev);
1531
1532 struct dentry *gpu_root = platform->debugfs;
1533 struct dentry *d;
1534
1535 gk20a_dbg(gpu_dbg_info, "g=%p", g);
1536
1537 d = debugfs_create_file(
1538 "arb_stats",
1539 S_IRUGO,
1540 gpu_root,
1541 g,
1542 &nvgpu_clk_arb_stats_fops);
1543 if (!d)
1544 return -ENOMEM;
1545
1546 return 0;
1547}
1548#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-08-26 09:00:01 -0400