summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/linux/clk_arb.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/clk_arb.c')
-rw-r--r--drivers/gpu/nvgpu/common/linux/clk_arb.c2121
1 files changed, 2121 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/clk_arb.c b/drivers/gpu/nvgpu/common/linux/clk_arb.c
new file mode 100644
index 00000000..ff26f567
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/clk_arb.c
@@ -0,0 +1,2121 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23
24#include <linux/cdev.h>
25#include <linux/file.h>
26#include <linux/anon_inodes.h>
27#include <linux/rculist.h>
28#include <linux/llist.h>
29#include <linux/uaccess.h>
30#include <linux/poll.h>
31#ifdef CONFIG_DEBUG_FS
32#include <linux/debugfs.h>
33#endif
34
35#include <nvgpu/bitops.h>
36#include <nvgpu/lock.h>
37#include <nvgpu/kmem.h>
38#include <nvgpu/atomic.h>
39#include <nvgpu/bug.h>
40#include <nvgpu/kref.h>
41#include <nvgpu/log.h>
42#include <nvgpu/barrier.h>
43#include <nvgpu/cond.h>
44#include <nvgpu/clk_arb.h>
45
46#include "gk20a/gk20a.h"
47#include "clk/clk.h"
48#include "pstate/pstate.h"
49#include "lpwr/lpwr.h"
50#include "volt/volt.h"
51
52#ifdef CONFIG_DEBUG_FS
53#include "common/linux/os_linux.h"
54#endif
55
56#define MAX_F_POINTS 256
57#define DEFAULT_EVENT_NUMBER 32
58
59struct nvgpu_clk_dev;
60struct nvgpu_clk_arb_target;
61struct nvgpu_clk_notification_queue;
62
63#ifdef CONFIG_DEBUG_FS
64static int nvgpu_clk_arb_debugfs_init(struct gk20a *g);
65#endif
66
67static int nvgpu_clk_arb_release_event_dev(struct inode *inode,
68 struct file *filp);
69static int nvgpu_clk_arb_release_completion_dev(struct inode *inode,
70 struct file *filp);
71static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait);
72static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf,
73 size_t size, loff_t *off);
74
75static long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd,
76 unsigned long arg);
77
78static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work);
79static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work);
80static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb);
81static void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount);
82static void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount);
83static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target,
84 u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv,
85 u32 voltuv_sram);
86static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb,
87 u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk,
88 u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram);
89static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev,
90 struct nvgpu_clk_arb_target *target,
91 u32 alarm_mask);
92static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm);
93static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm);
94
95static void nvgpu_clk_arb_queue_notification(struct gk20a *g,
96 struct nvgpu_clk_notification_queue *queue,
97 u32 alarm_mask);
98static int nvgpu_clk_notification_queue_alloc(struct gk20a *g,
99 struct nvgpu_clk_notification_queue *queue,
100 size_t events_number);
101
102static void nvgpu_clk_notification_queue_free(struct gk20a *g,
103 struct nvgpu_clk_notification_queue *queue);
104
105#define VF_POINT_INVALID_PSTATE ~0U
106#define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b)))
107#define VF_POINT_GET_PSTATE(a) (((a)->pstates) ?\
108 __fls((a)->pstates) :\
109 VF_POINT_INVALID_PSTATE)
110#define VF_POINT_COMMON_PSTATE(a, b) (((a)->pstates & (b)->pstates) ?\
111 __fls((a)->pstates & (b)->pstates) :\
112 VF_POINT_INVALID_PSTATE)
113
114/* Local Alarms */
115#define EVENT(alarm) (0x1UL << NVGPU_GPU_EVENT_##alarm)
116
117#define LOCAL_ALARM_MASK (EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE) | \
118 EVENT(VF_UPDATE))
119
120#define _WRAPGTEQ(a, b) ((a-b) > 0)
121
122struct nvgpu_clk_notification {
123 u32 notification;
124 u64 timestamp;
125};
126
127struct nvgpu_clk_notification_queue {
128 u32 size;
129 nvgpu_atomic_t head;
130 nvgpu_atomic_t tail;
131 struct nvgpu_clk_notification *notifications;
132};
133
134struct nvgpu_clk_vf_point {
135 u16 pstates;
136 union {
137 struct {
138 u16 gpc_mhz;
139 u16 sys_mhz;
140 u16 xbar_mhz;
141 };
142 u16 mem_mhz;
143 };
144 u32 uvolt;
145 u32 uvolt_sram;
146};
147
148struct nvgpu_clk_vf_table {
149 u32 mclk_num_points;
150 struct nvgpu_clk_vf_point *mclk_points;
151 u32 gpc2clk_num_points;
152 struct nvgpu_clk_vf_point *gpc2clk_points;
153};
154#ifdef CONFIG_DEBUG_FS
155struct nvgpu_clk_arb_debug {
156 s64 switch_max;
157 s64 switch_min;
158 u64 switch_num;
159 s64 switch_avg;
160 s64 switch_std;
161};
162#endif
163
164struct nvgpu_clk_arb_target {
165 u16 mclk;
166 u16 gpc2clk;
167 u32 pstate;
168};
169
170struct nvgpu_clk_arb {
171 struct nvgpu_spinlock sessions_lock;
172 struct nvgpu_spinlock users_lock;
173
174 struct nvgpu_mutex pstate_lock;
175 struct list_head users;
176 struct list_head sessions;
177 struct llist_head requests;
178
179 struct gk20a *g;
180 int status;
181
182 struct nvgpu_clk_arb_target actual_pool[2];
183 struct nvgpu_clk_arb_target *actual;
184
185 u16 gpc2clk_default_mhz;
186 u16 mclk_default_mhz;
187 u32 voltuv_actual;
188
189 u16 gpc2clk_min, gpc2clk_max;
190 u16 mclk_min, mclk_max;
191
192 struct work_struct update_fn_work;
193 struct workqueue_struct *update_work_queue;
194 struct work_struct vf_table_fn_work;
195 struct workqueue_struct *vf_table_work_queue;
196
197 struct nvgpu_cond request_wq;
198
199 struct nvgpu_clk_vf_table *current_vf_table;
200 struct nvgpu_clk_vf_table vf_table_pool[2];
201 u32 vf_table_index;
202
203 u16 *mclk_f_points;
204 nvgpu_atomic_t req_nr;
205
206 u32 mclk_f_numpoints;
207 u16 *gpc2clk_f_points;
208 u32 gpc2clk_f_numpoints;
209
210 nvgpu_atomic64_t alarm_mask;
211 struct nvgpu_clk_notification_queue notification_queue;
212
213#ifdef CONFIG_DEBUG_FS
214 struct nvgpu_clk_arb_debug debug_pool[2];
215 struct nvgpu_clk_arb_debug *debug;
216 bool debugfs_set;
217#endif
218};
219
220struct nvgpu_clk_dev {
221 struct nvgpu_clk_session *session;
222 union {
223 struct list_head link;
224 struct llist_node node;
225 };
226 struct nvgpu_cond readout_wq;
227 nvgpu_atomic_t poll_mask;
228 u16 gpc2clk_target_mhz;
229 u16 mclk_target_mhz;
230 u32 alarms_reported;
231 nvgpu_atomic_t enabled_mask;
232 struct nvgpu_clk_notification_queue queue;
233 u32 arb_queue_head;
234 struct nvgpu_ref refcount;
235};
236
237struct nvgpu_clk_session {
238 bool zombie;
239 struct gk20a *g;
240 struct nvgpu_ref refcount;
241 struct list_head link;
242 struct llist_head targets;
243
244 struct nvgpu_clk_arb_target target_pool[2];
245 struct nvgpu_clk_arb_target *target;
246};
247
248static const struct file_operations completion_dev_ops = {
249 .owner = THIS_MODULE,
250 .release = nvgpu_clk_arb_release_completion_dev,
251 .poll = nvgpu_clk_arb_poll_dev,
252};
253
254static const struct file_operations event_dev_ops = {
255 .owner = THIS_MODULE,
256 .release = nvgpu_clk_arb_release_event_dev,
257 .poll = nvgpu_clk_arb_poll_dev,
258 .read = nvgpu_clk_arb_read_event_dev,
259#ifdef CONFIG_COMPAT
260 .compat_ioctl = nvgpu_clk_arb_ioctl_event_dev,
261#endif
262 .unlocked_ioctl = nvgpu_clk_arb_ioctl_event_dev,
263};
264
265static int nvgpu_clk_notification_queue_alloc(struct gk20a *g,
266 struct nvgpu_clk_notification_queue *queue,
267 size_t events_number) {
268 queue->notifications = nvgpu_kcalloc(g, events_number,
269 sizeof(struct nvgpu_clk_notification));
270 if (!queue->notifications)
271 return -ENOMEM;
272 queue->size = events_number;
273
274 nvgpu_atomic_set(&queue->head, 0);
275 nvgpu_atomic_set(&queue->tail, 0);
276
277 return 0;
278}
279
280static void nvgpu_clk_notification_queue_free(struct gk20a *g,
281 struct nvgpu_clk_notification_queue *queue) {
282 nvgpu_kfree(g, queue->notifications);
283 queue->size = 0;
284 nvgpu_atomic_set(&queue->head, 0);
285 nvgpu_atomic_set(&queue->tail, 0);
286}
287
288int nvgpu_clk_arb_init_arbiter(struct gk20a *g)
289{
290 struct nvgpu_clk_arb *arb;
291 u16 default_mhz;
292 int err;
293 int index;
294 struct nvgpu_clk_vf_table *table;
295
296 gk20a_dbg_fn("");
297
298 if (!g->ops.clk_arb.get_arbiter_clk_domains)
299 return 0;
300
301 arb = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_arb));
302 if (!arb)
303 return -ENOMEM;
304
305 err = nvgpu_mutex_init(&arb->pstate_lock);
306 if (err)
307 goto mutex_fail;
308 nvgpu_spinlock_init(&arb->sessions_lock);
309 nvgpu_spinlock_init(&arb->users_lock);
310
311 arb->mclk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16));
312 if (!arb->mclk_f_points) {
313 err = -ENOMEM;
314 goto init_fail;
315 }
316
317 arb->gpc2clk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16));
318 if (!arb->gpc2clk_f_points) {
319 err = -ENOMEM;
320 goto init_fail;
321 }
322
323 for (index = 0; index < 2; index++) {
324 table = &arb->vf_table_pool[index];
325 table->gpc2clk_num_points = MAX_F_POINTS;
326 table->mclk_num_points = MAX_F_POINTS;
327
328 table->gpc2clk_points = nvgpu_kcalloc(g, MAX_F_POINTS,
329 sizeof(struct nvgpu_clk_vf_point));
330 if (!table->gpc2clk_points) {
331 err = -ENOMEM;
332 goto init_fail;
333 }
334
335
336 table->mclk_points = nvgpu_kcalloc(g, MAX_F_POINTS,
337 sizeof(struct nvgpu_clk_vf_point));
338 if (!table->mclk_points) {
339 err = -ENOMEM;
340 goto init_fail;
341 }
342 }
343
344 g->clk_arb = arb;
345 arb->g = g;
346
347 err = g->ops.clk_arb.get_arbiter_clk_default(g,
348 CTRL_CLK_DOMAIN_MCLK, &default_mhz);
349 if (err < 0) {
350 err = -EINVAL;
351 goto init_fail;
352 }
353
354 arb->mclk_default_mhz = default_mhz;
355
356 err = g->ops.clk_arb.get_arbiter_clk_default(g,
357 CTRL_CLK_DOMAIN_GPC2CLK, &default_mhz);
358 if (err < 0) {
359 err = -EINVAL;
360 goto init_fail;
361 }
362
363 arb->gpc2clk_default_mhz = default_mhz;
364
365 arb->actual = &arb->actual_pool[0];
366
367 nvgpu_atomic_set(&arb->req_nr, 0);
368
369 nvgpu_atomic64_set(&arb->alarm_mask, 0);
370 err = nvgpu_clk_notification_queue_alloc(g, &arb->notification_queue,
371 DEFAULT_EVENT_NUMBER);
372 if (err < 0)
373 goto init_fail;
374
375 INIT_LIST_HEAD_RCU(&arb->users);
376 INIT_LIST_HEAD_RCU(&arb->sessions);
377 init_llist_head(&arb->requests);
378
379 nvgpu_cond_init(&arb->request_wq);
380 arb->vf_table_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1,
381 "vf_table_update");
382 arb->update_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1,
383 "arbiter_update");
384
385
386 INIT_WORK(&arb->vf_table_fn_work, nvgpu_clk_arb_run_vf_table_cb);
387
388 INIT_WORK(&arb->update_fn_work, nvgpu_clk_arb_run_arbiter_cb);
389
390#ifdef CONFIG_DEBUG_FS
391 arb->debug = &arb->debug_pool[0];
392
393 if (!arb->debugfs_set) {
394 if (nvgpu_clk_arb_debugfs_init(g))
395 arb->debugfs_set = true;
396 }
397#endif
398 err = clk_vf_point_cache(g);
399 if (err < 0)
400 goto init_fail;
401
402 err = nvgpu_clk_arb_update_vf_table(arb);
403 if (err < 0)
404 goto init_fail;
405 do {
406 /* Check that first run is completed */
407 nvgpu_smp_mb();
408 NVGPU_COND_WAIT_INTERRUPTIBLE(&arb->request_wq,
409 nvgpu_atomic_read(&arb->req_nr), 0);
410 } while (!nvgpu_atomic_read(&arb->req_nr));
411
412
413 return arb->status;
414
415init_fail:
416 nvgpu_kfree(g, arb->gpc2clk_f_points);
417 nvgpu_kfree(g, arb->mclk_f_points);
418
419 for (index = 0; index < 2; index++) {
420 nvgpu_kfree(g, arb->vf_table_pool[index].gpc2clk_points);
421 nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points);
422 }
423
424 nvgpu_mutex_destroy(&arb->pstate_lock);
425
426mutex_fail:
427 nvgpu_kfree(g, arb);
428
429 return err;
430}
431
432void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm)
433{
434 struct nvgpu_clk_arb *arb = g->clk_arb;
435
436 nvgpu_clk_arb_set_global_alarm(g, alarm);
437 if (arb->update_work_queue)
438 queue_work(arb->update_work_queue, &arb->update_fn_work);
439}
440
441static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm)
442{
443 struct nvgpu_clk_arb *arb = g->clk_arb;
444
445 u64 current_mask;
446 u32 refcnt;
447 u32 alarm_mask;
448 u64 new_mask;
449
450 do {
451 current_mask = nvgpu_atomic64_read(&arb->alarm_mask);
452 /* atomic operations are strong so they do not need masks */
453
454 refcnt = ((u32) (current_mask >> 32)) + 1;
455 alarm_mask = (u32) (current_mask & ~alarm);
456 new_mask = ((u64) refcnt << 32) | alarm_mask;
457
458 } while (unlikely(current_mask !=
459 (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask,
460 current_mask, new_mask)));
461}
462
463static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm)
464{
465 struct nvgpu_clk_arb *arb = g->clk_arb;
466
467 u64 current_mask;
468 u32 refcnt;
469 u32 alarm_mask;
470 u64 new_mask;
471
472 do {
473 current_mask = nvgpu_atomic64_read(&arb->alarm_mask);
474 /* atomic operations are strong so they do not need masks */
475
476 refcnt = ((u32) (current_mask >> 32)) + 1;
477 alarm_mask = (u32) (current_mask & ~0) | alarm;
478 new_mask = ((u64) refcnt << 32) | alarm_mask;
479
480 } while (unlikely(current_mask !=
481 (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask,
482 current_mask, new_mask)));
483
484 nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm);
485
486}
487
488void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g)
489{
490 struct nvgpu_clk_arb *arb = g->clk_arb;
491 int index;
492
493 if (arb) {
494 cancel_work_sync(&arb->vf_table_fn_work);
495 destroy_workqueue(arb->vf_table_work_queue);
496 arb->vf_table_work_queue = NULL;
497
498 cancel_work_sync(&arb->update_fn_work);
499 destroy_workqueue(arb->update_work_queue);
500 arb->update_work_queue = NULL;
501
502 nvgpu_kfree(g, arb->gpc2clk_f_points);
503 nvgpu_kfree(g, arb->mclk_f_points);
504
505 for (index = 0; index < 2; index++) {
506 nvgpu_kfree(g,
507 arb->vf_table_pool[index].gpc2clk_points);
508 nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points);
509 }
510 nvgpu_mutex_destroy(&g->clk_arb->pstate_lock);
511 nvgpu_kfree(g, g->clk_arb);
512 g->clk_arb = NULL;
513 }
514}
515
516static int nvgpu_clk_arb_install_fd(struct gk20a *g,
517 struct nvgpu_clk_session *session,
518 const struct file_operations *fops,
519 struct nvgpu_clk_dev **_dev)
520{
521 struct file *file;
522 int fd;
523 int err;
524 int status;
525 char name[64];
526 struct nvgpu_clk_dev *dev;
527
528 gk20a_dbg_fn("");
529
530 dev = nvgpu_kzalloc(g, sizeof(*dev));
531 if (!dev)
532 return -ENOMEM;
533
534 status = nvgpu_clk_notification_queue_alloc(g, &dev->queue,
535 DEFAULT_EVENT_NUMBER);
536 if (status < 0) {
537 err = status;
538 goto fail;
539 }
540
541 fd = get_unused_fd_flags(O_RDWR);
542 if (fd < 0) {
543 err = fd;
544 goto fail;
545 }
546
547 snprintf(name, sizeof(name), "%s-clk-fd%d", g->name, fd);
548 file = anon_inode_getfile(name, fops, dev, O_RDWR);
549 if (IS_ERR(file)) {
550 err = PTR_ERR(file);
551 goto fail_fd;
552 }
553
554 fd_install(fd, file);
555
556 nvgpu_cond_init(&dev->readout_wq);
557
558 nvgpu_atomic_set(&dev->poll_mask, 0);
559
560 dev->session = session;
561 nvgpu_ref_init(&dev->refcount);
562
563 nvgpu_ref_get(&session->refcount);
564
565 *_dev = dev;
566
567 return fd;
568
569fail_fd:
570 put_unused_fd(fd);
571fail:
572 nvgpu_kfree(g, dev);
573
574 return err;
575}
576
577int nvgpu_clk_arb_init_session(struct gk20a *g,
578 struct nvgpu_clk_session **_session)
579{
580 struct nvgpu_clk_arb *arb = g->clk_arb;
581 struct nvgpu_clk_session *session = *(_session);
582
583 gk20a_dbg_fn("");
584
585 if (!g->ops.clk_arb.get_arbiter_clk_domains)
586 return 0;
587
588 session = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_session));
589 if (!session)
590 return -ENOMEM;
591 session->g = g;
592
593 nvgpu_ref_init(&session->refcount);
594
595 session->zombie = false;
596 session->target_pool[0].pstate = CTRL_PERF_PSTATE_P8;
597 /* make sure that the initialization of the pool is visible
598 * before the update
599 */
600 nvgpu_smp_wmb();
601 session->target = &session->target_pool[0];
602
603 init_llist_head(&session->targets);
604
605 nvgpu_spinlock_acquire(&arb->sessions_lock);
606 list_add_tail_rcu(&session->link, &arb->sessions);
607 nvgpu_spinlock_release(&arb->sessions_lock);
608
609 *_session = session;
610
611 return 0;
612}
613
614static void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount)
615{
616 struct nvgpu_clk_dev *dev = container_of(refcount,
617 struct nvgpu_clk_dev, refcount);
618 struct nvgpu_clk_session *session = dev->session;
619
620 nvgpu_kfree(session->g, dev);
621}
622
623static void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount)
624{
625 struct nvgpu_clk_session *session = container_of(refcount,
626 struct nvgpu_clk_session, refcount);
627 struct nvgpu_clk_arb *arb = session->g->clk_arb;
628 struct gk20a *g = session->g;
629 struct nvgpu_clk_dev *dev, *tmp;
630 struct llist_node *head;
631
632 gk20a_dbg_fn("");
633
634 if (arb) {
635 nvgpu_spinlock_acquire(&arb->sessions_lock);
636 list_del_rcu(&session->link);
637 nvgpu_spinlock_release(&arb->sessions_lock);
638 }
639
640 head = llist_del_all(&session->targets);
641 llist_for_each_entry_safe(dev, tmp, head, node) {
642 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
643 }
644 synchronize_rcu();
645 nvgpu_kfree(g, session);
646}
647
648void nvgpu_clk_arb_release_session(struct gk20a *g,
649 struct nvgpu_clk_session *session)
650{
651 struct nvgpu_clk_arb *arb = g->clk_arb;
652
653 gk20a_dbg_fn("");
654
655 session->zombie = true;
656 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
657 if (arb && arb->update_work_queue)
658 queue_work(arb->update_work_queue, &arb->update_fn_work);
659}
660
661int nvgpu_clk_arb_install_event_fd(struct gk20a *g,
662 struct nvgpu_clk_session *session, int *event_fd, u32 alarm_mask)
663{
664 struct nvgpu_clk_arb *arb = g->clk_arb;
665 struct nvgpu_clk_dev *dev;
666 int fd;
667
668 gk20a_dbg_fn("");
669
670 fd = nvgpu_clk_arb_install_fd(g, session, &event_dev_ops, &dev);
671 if (fd < 0)
672 return fd;
673
674 /* TODO: alarm mask needs to be set to default value to prevent
675 * failures of legacy tests. This will be removed when sanity is
676 * updated
677 */
678 if (alarm_mask)
679 nvgpu_atomic_set(&dev->enabled_mask, alarm_mask);
680 else
681 nvgpu_atomic_set(&dev->enabled_mask, EVENT(VF_UPDATE));
682
683 dev->arb_queue_head = nvgpu_atomic_read(&arb->notification_queue.head);
684
685 nvgpu_spinlock_acquire(&arb->users_lock);
686 list_add_tail_rcu(&dev->link, &arb->users);
687 nvgpu_spinlock_release(&arb->users_lock);
688
689 *event_fd = fd;
690
691 return 0;
692}
693
694int nvgpu_clk_arb_install_request_fd(struct gk20a *g,
695 struct nvgpu_clk_session *session, int *request_fd)
696{
697 struct nvgpu_clk_dev *dev;
698 int fd;
699
700 gk20a_dbg_fn("");
701
702 fd = nvgpu_clk_arb_install_fd(g, session, &completion_dev_ops, &dev);
703 if (fd < 0)
704 return fd;
705
706 *request_fd = fd;
707
708 return 0;
709}
710
711static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb)
712{
713 struct gk20a *g = arb->g;
714 struct nvgpu_clk_vf_table *table;
715
716 u32 i, j;
717 int status = -EINVAL;
718 u32 gpc2clk_voltuv = 0, mclk_voltuv = 0;
719 u32 gpc2clk_voltuv_sram = 0, mclk_voltuv_sram = 0;
720 u16 clk_cur;
721 u32 num_points;
722
723 struct clk_set_info *p5_info, *p0_info;
724
725
726 table = NV_ACCESS_ONCE(arb->current_vf_table);
727 /* make flag visible when all data has resolved in the tables */
728 nvgpu_smp_rmb();
729
730 table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] :
731 &arb->vf_table_pool[0];
732
733 /* Get allowed memory ranges */
734 if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_GPC2CLK,
735 &arb->gpc2clk_min,
736 &arb->gpc2clk_max) < 0) {
737 nvgpu_err(g, "failed to fetch GPC2CLK range");
738 goto exit_vf_table;
739 }
740 if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_MCLK,
741 &arb->mclk_min,
742 &arb->mclk_max) < 0) {
743 nvgpu_err(g, "failed to fetch MCLK range");
744 goto exit_vf_table;
745 }
746
747 table->gpc2clk_num_points = MAX_F_POINTS;
748 table->mclk_num_points = MAX_F_POINTS;
749
750 if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_GPC2CLK,
751 &table->gpc2clk_num_points, arb->gpc2clk_f_points)) {
752 nvgpu_err(g, "failed to fetch GPC2CLK frequency points");
753 goto exit_vf_table;
754 }
755
756 if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_MCLK,
757 &table->mclk_num_points, arb->mclk_f_points)) {
758 nvgpu_err(g, "failed to fetch MCLK frequency points");
759 goto exit_vf_table;
760 }
761 if (!table->mclk_num_points || !table->gpc2clk_num_points) {
762 nvgpu_err(g, "empty queries to f points mclk %d gpc2clk %d",
763 table->mclk_num_points, table->gpc2clk_num_points);
764 status = -EINVAL;
765 goto exit_vf_table;
766 }
767
768 memset(table->mclk_points, 0,
769 table->mclk_num_points*sizeof(struct nvgpu_clk_vf_point));
770 memset(table->gpc2clk_points, 0,
771 table->gpc2clk_num_points*sizeof(struct nvgpu_clk_vf_point));
772
773 p5_info = pstate_get_clk_set_info(g,
774 CTRL_PERF_PSTATE_P5, clkwhich_mclk);
775 if (!p5_info) {
776 nvgpu_err(g, "failed to get MCLK P5 info");
777 goto exit_vf_table;
778 }
779 p0_info = pstate_get_clk_set_info(g,
780 CTRL_PERF_PSTATE_P0, clkwhich_mclk);
781 if (!p0_info) {
782 nvgpu_err(g, "failed to get MCLK P0 info");
783 goto exit_vf_table;
784 }
785
786 for (i = 0, j = 0, num_points = 0, clk_cur = 0;
787 i < table->mclk_num_points; i++) {
788
789 if ((arb->mclk_f_points[i] >= arb->mclk_min) &&
790 (arb->mclk_f_points[i] <= arb->mclk_max) &&
791 (arb->mclk_f_points[i] != clk_cur)) {
792
793 table->mclk_points[j].mem_mhz = arb->mclk_f_points[i];
794 mclk_voltuv = mclk_voltuv_sram = 0;
795
796 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK,
797 &table->mclk_points[j].mem_mhz, &mclk_voltuv,
798 CTRL_VOLT_DOMAIN_LOGIC);
799 if (status < 0) {
800 nvgpu_err(g,
801 "failed to get MCLK LOGIC voltage");
802 goto exit_vf_table;
803 }
804 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK,
805 &table->mclk_points[j].mem_mhz,
806 &mclk_voltuv_sram,
807 CTRL_VOLT_DOMAIN_SRAM);
808 if (status < 0) {
809 nvgpu_err(g, "failed to get MCLK SRAM voltage");
810 goto exit_vf_table;
811 }
812
813 table->mclk_points[j].uvolt = mclk_voltuv;
814 table->mclk_points[j].uvolt_sram = mclk_voltuv_sram;
815 clk_cur = table->mclk_points[j].mem_mhz;
816
817 if ((clk_cur >= p5_info->min_mhz) &&
818 (clk_cur <= p5_info->max_mhz))
819 VF_POINT_SET_PSTATE_SUPPORTED(
820 &table->mclk_points[j],
821 CTRL_PERF_PSTATE_P5);
822 if ((clk_cur >= p0_info->min_mhz) &&
823 (clk_cur <= p0_info->max_mhz))
824 VF_POINT_SET_PSTATE_SUPPORTED(
825 &table->mclk_points[j],
826 CTRL_PERF_PSTATE_P0);
827
828 j++;
829 num_points++;
830
831 }
832 }
833 table->mclk_num_points = num_points;
834
835 p5_info = pstate_get_clk_set_info(g,
836 CTRL_PERF_PSTATE_P5, clkwhich_gpc2clk);
837 if (!p5_info) {
838 status = -EINVAL;
839 nvgpu_err(g, "failed to get GPC2CLK P5 info");
840 goto exit_vf_table;
841 }
842
843 p0_info = pstate_get_clk_set_info(g,
844 CTRL_PERF_PSTATE_P0, clkwhich_gpc2clk);
845 if (!p0_info) {
846 status = -EINVAL;
847 nvgpu_err(g, "failed to get GPC2CLK P0 info");
848 goto exit_vf_table;
849 }
850
851 /* GPC2CLK needs to be checked in two passes. The first determines the
852 * relationships between GPC2CLK, SYS2CLK and XBAR2CLK, while the
853 * second verifies that the clocks minimum is satisfied and sets
854 * the voltages
855 */
856 for (i = 0, j = 0, num_points = 0, clk_cur = 0;
857 i < table->gpc2clk_num_points; i++) {
858 struct set_fll_clk setfllclk;
859
860 if ((arb->gpc2clk_f_points[i] >= arb->gpc2clk_min) &&
861 (arb->gpc2clk_f_points[i] <= arb->gpc2clk_max) &&
862 (arb->gpc2clk_f_points[i] != clk_cur)) {
863
864 table->gpc2clk_points[j].gpc_mhz =
865 arb->gpc2clk_f_points[i];
866 setfllclk.gpc2clkmhz = arb->gpc2clk_f_points[i];
867 status = clk_get_fll_clks(g, &setfllclk);
868 if (status < 0) {
869 nvgpu_err(g,
870 "failed to get GPC2CLK slave clocks");
871 goto exit_vf_table;
872 }
873
874 table->gpc2clk_points[j].sys_mhz =
875 setfllclk.sys2clkmhz;
876 table->gpc2clk_points[j].xbar_mhz =
877 setfllclk.xbar2clkmhz;
878
879 clk_cur = table->gpc2clk_points[j].gpc_mhz;
880
881 if ((clk_cur >= p5_info->min_mhz) &&
882 (clk_cur <= p5_info->max_mhz))
883 VF_POINT_SET_PSTATE_SUPPORTED(
884 &table->gpc2clk_points[j],
885 CTRL_PERF_PSTATE_P5);
886 if ((clk_cur >= p0_info->min_mhz) &&
887 (clk_cur <= p0_info->max_mhz))
888 VF_POINT_SET_PSTATE_SUPPORTED(
889 &table->gpc2clk_points[j],
890 CTRL_PERF_PSTATE_P0);
891
892 j++;
893 num_points++;
894 }
895 }
896 table->gpc2clk_num_points = num_points;
897
898 /* Second pass */
899 for (i = 0, j = 0; i < table->gpc2clk_num_points; i++) {
900
901 u16 alt_gpc2clk = table->gpc2clk_points[i].gpc_mhz;
902
903 gpc2clk_voltuv = gpc2clk_voltuv_sram = 0;
904
905 /* Check sysclk */
906 p5_info = pstate_get_clk_set_info(g,
907 VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]),
908 clkwhich_sys2clk);
909 if (!p5_info) {
910 status = -EINVAL;
911 nvgpu_err(g, "failed to get SYS2CLK P5 info");
912 goto exit_vf_table;
913 }
914
915 /* sys2clk below clk min, need to find correct clock */
916 if (table->gpc2clk_points[i].sys_mhz < p5_info->min_mhz) {
917 for (j = i + 1; j < table->gpc2clk_num_points; j++) {
918
919 if (table->gpc2clk_points[j].sys_mhz >=
920 p5_info->min_mhz) {
921
922
923 table->gpc2clk_points[i].sys_mhz =
924 p5_info->min_mhz;
925
926 alt_gpc2clk = alt_gpc2clk <
927 table->gpc2clk_points[j].
928 gpc_mhz ?
929 table->gpc2clk_points[j].
930 gpc_mhz :
931 alt_gpc2clk;
932 break;
933 }
934 }
935 /* no VF exists that satisfies condition */
936 if (j == table->gpc2clk_num_points) {
937 nvgpu_err(g, "NO SYS2CLK VF point possible");
938 status = -EINVAL;
939 goto exit_vf_table;
940 }
941 }
942
943 /* Check xbarclk */
944 p5_info = pstate_get_clk_set_info(g,
945 VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]),
946 clkwhich_xbar2clk);
947 if (!p5_info) {
948 status = -EINVAL;
949 nvgpu_err(g, "failed to get SYS2CLK P5 info");
950 goto exit_vf_table;
951 }
952
953 /* xbar2clk below clk min, need to find correct clock */
954 if (table->gpc2clk_points[i].xbar_mhz < p5_info->min_mhz) {
955 for (j = i; j < table->gpc2clk_num_points; j++) {
956 if (table->gpc2clk_points[j].xbar_mhz >=
957 p5_info->min_mhz) {
958
959 table->gpc2clk_points[i].xbar_mhz =
960 p5_info->min_mhz;
961
962 alt_gpc2clk = alt_gpc2clk <
963 table->gpc2clk_points[j].
964 gpc_mhz ?
965 table->gpc2clk_points[j].
966 gpc_mhz :
967 alt_gpc2clk;
968 break;
969 }
970 }
971 /* no VF exists that satisfies condition */
972 if (j == table->gpc2clk_num_points) {
973 status = -EINVAL;
974 nvgpu_err(g, "NO XBAR2CLK VF point possible");
975
976 goto exit_vf_table;
977 }
978 }
979
980 /* Calculate voltages */
981 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK,
982 &alt_gpc2clk, &gpc2clk_voltuv,
983 CTRL_VOLT_DOMAIN_LOGIC);
984 if (status < 0) {
985 nvgpu_err(g, "failed to get GPC2CLK LOGIC voltage");
986 goto exit_vf_table;
987 }
988
989 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK,
990 &alt_gpc2clk,
991 &gpc2clk_voltuv_sram,
992 CTRL_VOLT_DOMAIN_SRAM);
993 if (status < 0) {
994 nvgpu_err(g, "failed to get GPC2CLK SRAM voltage");
995 goto exit_vf_table;
996 }
997
998 table->gpc2clk_points[i].uvolt = gpc2clk_voltuv;
999 table->gpc2clk_points[i].uvolt_sram = gpc2clk_voltuv_sram;
1000 }
1001
1002 /* make table visible when all data has resolved in the tables */
1003 nvgpu_smp_wmb();
1004 xchg(&arb->current_vf_table, table);
1005
1006exit_vf_table:
1007
1008 if (status < 0)
1009 nvgpu_clk_arb_set_global_alarm(g,
1010 EVENT(ALARM_VF_TABLE_UPDATE_FAILED));
1011 if (arb->update_work_queue)
1012 queue_work(arb->update_work_queue, &arb->update_fn_work);
1013
1014 return status;
1015}
1016
1017void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g)
1018{
1019 struct nvgpu_clk_arb *arb = g->clk_arb;
1020
1021 if (arb->vf_table_work_queue)
1022 queue_work(arb->vf_table_work_queue, &arb->vf_table_fn_work);
1023}
1024
1025static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work)
1026{
1027 struct nvgpu_clk_arb *arb =
1028 container_of(work, struct nvgpu_clk_arb, vf_table_fn_work);
1029 struct gk20a *g = arb->g;
1030 u32 err;
1031
1032 /* get latest vf curve from pmu */
1033 err = clk_vf_point_cache(g);
1034 if (err) {
1035 nvgpu_err(g, "failed to cache VF table");
1036 nvgpu_clk_arb_set_global_alarm(g,
1037 EVENT(ALARM_VF_TABLE_UPDATE_FAILED));
1038 if (arb->update_work_queue)
1039 queue_work(arb->update_work_queue,
1040 &arb->update_fn_work);
1041
1042 return;
1043 }
1044 nvgpu_clk_arb_update_vf_table(arb);
1045}
1046
1047static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work)
1048{
1049 struct nvgpu_clk_arb *arb =
1050 container_of(work, struct nvgpu_clk_arb, update_fn_work);
1051 struct nvgpu_clk_session *session;
1052 struct nvgpu_clk_dev *dev;
1053 struct nvgpu_clk_dev *tmp;
1054 struct nvgpu_clk_arb_target *target, *actual;
1055 struct gk20a *g = arb->g;
1056 struct llist_node *head;
1057
1058 u32 pstate = VF_POINT_INVALID_PSTATE;
1059 u32 voltuv, voltuv_sram;
1060 bool mclk_set, gpc2clk_set;
1061 u32 nuvmin, nuvmin_sram;
1062
1063 u32 alarms_notified = 0;
1064 u32 current_alarm;
1065 int status = 0;
1066
1067 /* Temporary variables for checking target frequency */
1068 u16 gpc2clk_target, sys2clk_target, xbar2clk_target, mclk_target;
1069 u16 gpc2clk_session_target, mclk_session_target;
1070
1071#ifdef CONFIG_DEBUG_FS
1072 u64 t0, t1;
1073 struct nvgpu_clk_arb_debug *debug;
1074
1075#endif
1076
1077 gk20a_dbg_fn("");
1078
1079 /* bail out if gpu is down */
1080 if (nvgpu_atomic64_read(&arb->alarm_mask) & EVENT(ALARM_GPU_LOST))
1081 goto exit_arb;
1082
1083#ifdef CONFIG_DEBUG_FS
1084 g->ops.bus.read_ptimer(g, &t0);
1085#endif
1086
1087 /* Only one arbiter should be running */
1088 gpc2clk_target = 0;
1089 mclk_target = 0;
1090
1091 rcu_read_lock();
1092 list_for_each_entry_rcu(session, &arb->sessions, link) {
1093 if (!session->zombie) {
1094 mclk_set = false;
1095 gpc2clk_set = false;
1096 target = NV_ACCESS_ONCE(session->target) ==
1097 &session->target_pool[0] ?
1098 &session->target_pool[1] :
1099 &session->target_pool[0];
1100 /* Do not reorder pointer */
1101 nvgpu_smp_rmb();
1102 head = llist_del_all(&session->targets);
1103 if (head) {
1104
1105 /* Copy over state */
1106 target->mclk = session->target->mclk;
1107 target->gpc2clk = session->target->gpc2clk;
1108 /* Query the latest committed request */
1109 llist_for_each_entry_safe(dev, tmp, head,
1110 node) {
1111 if (!mclk_set && dev->mclk_target_mhz) {
1112 target->mclk =
1113 dev->mclk_target_mhz;
1114 mclk_set = true;
1115 }
1116 if (!gpc2clk_set &&
1117 dev->gpc2clk_target_mhz) {
1118 target->gpc2clk =
1119 dev->gpc2clk_target_mhz;
1120 gpc2clk_set = true;
1121 }
1122 nvgpu_ref_get(&dev->refcount);
1123 llist_add(&dev->node, &arb->requests);
1124 }
1125 /* Ensure target is updated before ptr sawp */
1126 nvgpu_smp_wmb();
1127 xchg(&session->target, target);
1128 }
1129
1130 mclk_target = mclk_target > session->target->mclk ?
1131 mclk_target : session->target->mclk;
1132
1133 gpc2clk_target =
1134 gpc2clk_target > session->target->gpc2clk ?
1135 gpc2clk_target : session->target->gpc2clk;
1136 }
1137 }
1138 rcu_read_unlock();
1139
1140 gpc2clk_target = (gpc2clk_target > 0) ? gpc2clk_target :
1141 arb->gpc2clk_default_mhz;
1142
1143 if (gpc2clk_target < arb->gpc2clk_min)
1144 gpc2clk_target = arb->gpc2clk_min;
1145
1146 if (gpc2clk_target > arb->gpc2clk_max)
1147 gpc2clk_target = arb->gpc2clk_max;
1148
1149 mclk_target = (mclk_target > 0) ? mclk_target :
1150 arb->mclk_default_mhz;
1151
1152 if (mclk_target < arb->mclk_min)
1153 mclk_target = arb->mclk_min;
1154
1155 if (mclk_target > arb->mclk_max)
1156 mclk_target = arb->mclk_max;
1157
1158 sys2clk_target = 0;
1159 xbar2clk_target = 0;
1160
1161 gpc2clk_session_target = gpc2clk_target;
1162 mclk_session_target = mclk_target;
1163
1164 /* Query the table for the closest vf point to program */
1165 pstate = nvgpu_clk_arb_find_vf_point(arb, &gpc2clk_target,
1166 &sys2clk_target, &xbar2clk_target, &mclk_target, &voltuv,
1167 &voltuv_sram, &nuvmin, &nuvmin_sram);
1168
1169 if (pstate == VF_POINT_INVALID_PSTATE) {
1170 arb->status = -EINVAL;
1171 /* make status visible */
1172 nvgpu_smp_mb();
1173 goto exit_arb;
1174 }
1175
1176 if ((gpc2clk_target < gpc2clk_session_target) ||
1177 (mclk_target < mclk_session_target))
1178 nvgpu_clk_arb_set_global_alarm(g,
1179 EVENT(ALARM_TARGET_VF_NOT_POSSIBLE));
1180
1181 if ((arb->actual->gpc2clk == gpc2clk_target) &&
1182 (arb->actual->mclk == mclk_target) &&
1183 (arb->voltuv_actual == voltuv)) {
1184 goto exit_arb;
1185 }
1186
1187 /* Program clocks */
1188 /* A change in both mclk of gpc2clk may require a change in voltage */
1189
1190 nvgpu_mutex_acquire(&arb->pstate_lock);
1191 status = nvgpu_lpwr_disable_pg(g, false);
1192
1193 status = clk_pmu_freq_controller_load(g, false,
1194 CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL);
1195 if (status < 0) {
1196 arb->status = status;
1197 nvgpu_mutex_release(&arb->pstate_lock);
1198
1199 /* make status visible */
1200 nvgpu_smp_mb();
1201 goto exit_arb;
1202 }
1203 status = volt_set_noiseaware_vmin(g, nuvmin, nuvmin_sram);
1204 if (status < 0) {
1205 arb->status = status;
1206 nvgpu_mutex_release(&arb->pstate_lock);
1207
1208 /* make status visible */
1209 nvgpu_smp_mb();
1210 goto exit_arb;
1211 }
1212
1213 status = nvgpu_clk_arb_change_vf_point(g, gpc2clk_target,
1214 sys2clk_target, xbar2clk_target, mclk_target, voltuv,
1215 voltuv_sram);
1216 if (status < 0) {
1217 arb->status = status;
1218 nvgpu_mutex_release(&arb->pstate_lock);
1219
1220 /* make status visible */
1221 nvgpu_smp_mb();
1222 goto exit_arb;
1223 }
1224
1225 status = clk_pmu_freq_controller_load(g, true,
1226 CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL);
1227 if (status < 0) {
1228 arb->status = status;
1229 nvgpu_mutex_release(&arb->pstate_lock);
1230
1231 /* make status visible */
1232 nvgpu_smp_mb();
1233 goto exit_arb;
1234 }
1235
1236 status = nvgpu_lwpr_mclk_change(g, pstate);
1237 if (status < 0) {
1238 arb->status = status;
1239 nvgpu_mutex_release(&arb->pstate_lock);
1240
1241 /* make status visible */
1242 nvgpu_smp_mb();
1243 goto exit_arb;
1244 }
1245
1246 actual = NV_ACCESS_ONCE(arb->actual) == &arb->actual_pool[0] ?
1247 &arb->actual_pool[1] : &arb->actual_pool[0];
1248
1249 /* do not reorder this pointer */
1250 nvgpu_smp_rmb();
1251 actual->gpc2clk = gpc2clk_target;
1252 actual->mclk = mclk_target;
1253 arb->voltuv_actual = voltuv;
1254 actual->pstate = pstate;
1255 arb->status = status;
1256
1257 /* Make changes visible to other threads */
1258 nvgpu_smp_wmb();
1259 xchg(&arb->actual, actual);
1260
1261 status = nvgpu_lpwr_enable_pg(g, false);
1262 if (status < 0) {
1263 arb->status = status;
1264 nvgpu_mutex_release(&arb->pstate_lock);
1265
1266 /* make status visible */
1267 nvgpu_smp_mb();
1268 goto exit_arb;
1269 }
1270
1271 /* status must be visible before atomic inc */
1272 nvgpu_smp_wmb();
1273 nvgpu_atomic_inc(&arb->req_nr);
1274
1275 /* Unlock pstate change for PG */
1276 nvgpu_mutex_release(&arb->pstate_lock);
1277
1278 /* VF Update complete */
1279 nvgpu_clk_arb_set_global_alarm(g, EVENT(VF_UPDATE));
1280
1281 nvgpu_cond_signal_interruptible(&arb->request_wq);
1282
1283#ifdef CONFIG_DEBUG_FS
1284 g->ops.bus.read_ptimer(g, &t1);
1285
1286 debug = arb->debug == &arb->debug_pool[0] ?
1287 &arb->debug_pool[1] : &arb->debug_pool[0];
1288
1289 memcpy(debug, arb->debug, sizeof(arb->debug_pool[0]));
1290 debug->switch_num++;
1291
1292 if (debug->switch_num == 1) {
1293 debug->switch_max = debug->switch_min =
1294 debug->switch_avg = (t1-t0)/1000;
1295 debug->switch_std = 0;
1296 } else {
1297 s64 prev_avg;
1298 s64 curr = (t1-t0)/1000;
1299
1300 debug->switch_max = curr > debug->switch_max ?
1301 curr : debug->switch_max;
1302 debug->switch_min = debug->switch_min ?
1303 (curr < debug->switch_min ?
1304 curr : debug->switch_min) : curr;
1305 prev_avg = debug->switch_avg;
1306 debug->switch_avg = (curr +
1307 (debug->switch_avg * (debug->switch_num-1))) /
1308 debug->switch_num;
1309 debug->switch_std +=
1310 (curr - debug->switch_avg) * (curr - prev_avg);
1311 }
1312 /* commit changes before exchanging debug pointer */
1313 nvgpu_smp_wmb();
1314 xchg(&arb->debug, debug);
1315#endif
1316
1317exit_arb:
1318 if (status < 0) {
1319 nvgpu_err(g, "Error in arbiter update");
1320 nvgpu_clk_arb_set_global_alarm(g,
1321 EVENT(ALARM_CLOCK_ARBITER_FAILED));
1322 }
1323
1324 current_alarm = (u32) nvgpu_atomic64_read(&arb->alarm_mask);
1325 /* notify completion for all requests */
1326 head = llist_del_all(&arb->requests);
1327 llist_for_each_entry_safe(dev, tmp, head, node) {
1328 nvgpu_atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM);
1329 nvgpu_cond_signal_interruptible(&dev->readout_wq);
1330 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1331 }
1332
1333 nvgpu_atomic_set(&arb->notification_queue.head,
1334 nvgpu_atomic_read(&arb->notification_queue.tail));
1335 /* notify event for all users */
1336 rcu_read_lock();
1337 list_for_each_entry_rcu(dev, &arb->users, link) {
1338 alarms_notified |=
1339 nvgpu_clk_arb_notify(dev, arb->actual, current_alarm);
1340 }
1341 rcu_read_unlock();
1342
1343 /* clear alarms */
1344 nvgpu_clk_arb_clear_global_alarm(g, alarms_notified &
1345 ~EVENT(ALARM_GPU_LOST));
1346}
1347
1348static void nvgpu_clk_arb_queue_notification(struct gk20a *g,
1349 struct nvgpu_clk_notification_queue *queue,
1350 u32 alarm_mask) {
1351
1352 u32 queue_index;
1353 u64 timestamp;
1354
1355 queue_index = (nvgpu_atomic_inc_return(&queue->tail)) % queue->size;
1356 /* get current timestamp */
1357 timestamp = (u64) sched_clock();
1358
1359 queue->notifications[queue_index].timestamp = timestamp;
1360 queue->notifications[queue_index].notification = alarm_mask;
1361
1362}
1363
1364static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev,
1365 struct nvgpu_clk_arb_target *target,
1366 u32 alarm) {
1367
1368 struct nvgpu_clk_session *session = dev->session;
1369 struct nvgpu_clk_arb *arb = session->g->clk_arb;
1370 struct nvgpu_clk_notification *notification;
1371
1372 u32 queue_alarm_mask = 0;
1373 u32 enabled_mask = 0;
1374 u32 new_alarms_reported = 0;
1375 u32 poll_mask = 0;
1376 u32 tail, head;
1377 u32 queue_index;
1378 size_t size;
1379 int index;
1380
1381 enabled_mask = nvgpu_atomic_read(&dev->enabled_mask);
1382 size = arb->notification_queue.size;
1383
1384 /* queue global arbiter notifications in buffer */
1385 do {
1386 tail = nvgpu_atomic_read(&arb->notification_queue.tail);
1387 /* copy items to the queue */
1388 queue_index = nvgpu_atomic_read(&dev->queue.tail);
1389 head = dev->arb_queue_head;
1390 head = (tail - head) < arb->notification_queue.size ?
1391 head : tail - arb->notification_queue.size;
1392
1393 for (index = head; _WRAPGTEQ(tail, index); index++) {
1394 u32 alarm_detected;
1395
1396 notification = &arb->notification_queue.
1397 notifications[(index+1) % size];
1398 alarm_detected =
1399 NV_ACCESS_ONCE(notification->notification);
1400
1401 if (!(enabled_mask & alarm_detected))
1402 continue;
1403
1404 queue_index++;
1405 dev->queue.notifications[
1406 queue_index % dev->queue.size].timestamp =
1407 NV_ACCESS_ONCE(notification->timestamp);
1408
1409 dev->queue.notifications[
1410 queue_index % dev->queue.size].notification =
1411 alarm_detected;
1412
1413 queue_alarm_mask |= alarm_detected;
1414 }
1415 } while (unlikely(nvgpu_atomic_read(&arb->notification_queue.tail) !=
1416 (int)tail));
1417
1418 nvgpu_atomic_set(&dev->queue.tail, queue_index);
1419 /* update the last notification we processed from global queue */
1420
1421 dev->arb_queue_head = tail;
1422
1423 /* Check if current session targets are met */
1424 if (enabled_mask & EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)) {
1425 if ((target->gpc2clk < session->target->gpc2clk)
1426 || (target->mclk < session->target->mclk)) {
1427
1428 poll_mask |= (POLLIN | POLLPRI);
1429 nvgpu_clk_arb_queue_notification(arb->g, &dev->queue,
1430 EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE));
1431 }
1432 }
1433
1434 /* Check if there is a new VF update */
1435 if (queue_alarm_mask & EVENT(VF_UPDATE))
1436 poll_mask |= (POLLIN | POLLRDNORM);
1437
1438 /* Notify sticky alarms that were not reported on previous run*/
1439 new_alarms_reported = (queue_alarm_mask |
1440 (alarm & ~dev->alarms_reported & queue_alarm_mask));
1441
1442 if (new_alarms_reported & ~LOCAL_ALARM_MASK) {
1443 /* check that we are not re-reporting */
1444 if (new_alarms_reported & EVENT(ALARM_GPU_LOST))
1445 poll_mask |= POLLHUP;
1446
1447 poll_mask |= (POLLIN | POLLPRI);
1448 /* On next run do not report global alarms that were already
1449 * reported, but report SHUTDOWN always
1450 */
1451 dev->alarms_reported = new_alarms_reported & ~LOCAL_ALARM_MASK &
1452 ~EVENT(ALARM_GPU_LOST);
1453 }
1454
1455 if (poll_mask) {
1456 nvgpu_atomic_set(&dev->poll_mask, poll_mask);
1457 nvgpu_cond_broadcast_interruptible(&dev->readout_wq);
1458 }
1459
1460 return new_alarms_reported;
1461}
1462
1463static int nvgpu_clk_arb_set_event_filter(struct nvgpu_clk_dev *dev,
1464 struct nvgpu_gpu_set_event_filter_args *args)
1465{
1466 u32 mask;
1467
1468 gk20a_dbg(gpu_dbg_fn, "");
1469
1470 if (args->flags)
1471 return -EINVAL;
1472
1473 if (args->size != 1)
1474 return -EINVAL;
1475
1476 if (copy_from_user(&mask, (void __user *) args->buffer,
1477 args->size * sizeof(u32)))
1478 return -EFAULT;
1479
1480 /* update alarm mask */
1481 nvgpu_atomic_set(&dev->enabled_mask, mask);
1482
1483 return 0;
1484}
1485
1486static long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd,
1487 unsigned long arg)
1488{
1489 struct nvgpu_clk_dev *dev = filp->private_data;
1490 struct gk20a *g = dev->session->g;
1491 u8 buf[NVGPU_EVENT_IOCTL_MAX_ARG_SIZE];
1492 int err = 0;
1493
1494 gk20a_dbg(gpu_dbg_fn, "nr=%d", _IOC_NR(cmd));
1495
1496 if ((_IOC_TYPE(cmd) != NVGPU_EVENT_IOCTL_MAGIC) || (_IOC_NR(cmd) == 0)
1497 || (_IOC_NR(cmd) > NVGPU_EVENT_IOCTL_LAST))
1498 return -EINVAL;
1499
1500 BUG_ON(_IOC_SIZE(cmd) > NVGPU_EVENT_IOCTL_MAX_ARG_SIZE);
1501
1502 memset(buf, 0, sizeof(buf));
1503 if (_IOC_DIR(cmd) & _IOC_WRITE) {
1504 if (copy_from_user(buf, (void __user *) arg, _IOC_SIZE(cmd)))
1505 return -EFAULT;
1506 }
1507
1508 switch (cmd) {
1509 case NVGPU_EVENT_IOCTL_SET_FILTER:
1510 err = nvgpu_clk_arb_set_event_filter(dev,
1511 (struct nvgpu_gpu_set_event_filter_args *)buf);
1512 break;
1513 default:
1514 nvgpu_warn(g, "unrecognized event ioctl cmd: 0x%x", cmd);
1515 err = -ENOTTY;
1516 }
1517
1518 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1519 err = copy_to_user((void __user *) arg, buf, _IOC_SIZE(cmd));
1520
1521 return err;
1522}
1523
1524int nvgpu_clk_arb_commit_request_fd(struct gk20a *g,
1525 struct nvgpu_clk_session *session, int request_fd)
1526{
1527 struct nvgpu_clk_arb *arb = g->clk_arb;
1528 struct nvgpu_clk_dev *dev;
1529 struct fd fd;
1530 int err = 0;
1531
1532 gk20a_dbg_fn("");
1533
1534 fd = fdget(request_fd);
1535 if (!fd.file)
1536 return -EINVAL;
1537
1538 if (fd.file->f_op != &completion_dev_ops) {
1539 err = -EINVAL;
1540 goto fdput_fd;
1541 }
1542
1543 dev = (struct nvgpu_clk_dev *) fd.file->private_data;
1544
1545 if (!dev || dev->session != session) {
1546 err = -EINVAL;
1547 goto fdput_fd;
1548 }
1549 nvgpu_ref_get(&dev->refcount);
1550 llist_add(&dev->node, &session->targets);
1551 if (arb->update_work_queue)
1552 queue_work(arb->update_work_queue, &arb->update_fn_work);
1553
1554fdput_fd:
1555 fdput(fd);
1556 return err;
1557}
1558
1559static inline u32 __pending_event(struct nvgpu_clk_dev *dev,
1560 struct nvgpu_gpu_event_info *info) {
1561
1562 u32 tail, head;
1563 u32 events = 0;
1564 struct nvgpu_clk_notification *p_notif;
1565
1566 tail = nvgpu_atomic_read(&dev->queue.tail);
1567 head = nvgpu_atomic_read(&dev->queue.head);
1568
1569 head = (tail - head) < dev->queue.size ? head : tail - dev->queue.size;
1570
1571 if (_WRAPGTEQ(tail, head) && info) {
1572 head++;
1573 p_notif = &dev->queue.notifications[head % dev->queue.size];
1574 events |= p_notif->notification;
1575 info->event_id = ffs(events) - 1;
1576 info->timestamp = p_notif->timestamp;
1577 nvgpu_atomic_set(&dev->queue.head, head);
1578 }
1579
1580 return events;
1581}
1582
1583static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf,
1584 size_t size, loff_t *off)
1585{
1586 struct nvgpu_clk_dev *dev = filp->private_data;
1587 struct nvgpu_gpu_event_info info;
1588 ssize_t err;
1589
1590 gk20a_dbg_fn("filp=%p, buf=%p, size=%zu", filp, buf, size);
1591
1592 if ((size - *off) < sizeof(info))
1593 return 0;
1594
1595 memset(&info, 0, sizeof(info));
1596 /* Get the oldest event from the queue */
1597 while (!__pending_event(dev, &info)) {
1598 if (filp->f_flags & O_NONBLOCK)
1599 return -EAGAIN;
1600 err = NVGPU_COND_WAIT_INTERRUPTIBLE(&dev->readout_wq,
1601 __pending_event(dev, &info), 0);
1602 if (err)
1603 return err;
1604 if (info.timestamp)
1605 break;
1606 }
1607
1608 if (copy_to_user(buf + *off, &info, sizeof(info)))
1609 return -EFAULT;
1610
1611 return sizeof(info);
1612}
1613
1614static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait)
1615{
1616 struct nvgpu_clk_dev *dev = filp->private_data;
1617
1618 gk20a_dbg_fn("");
1619
1620 poll_wait(filp, &dev->readout_wq.wq, wait);
1621 return nvgpu_atomic_xchg(&dev->poll_mask, 0);
1622}
1623
1624static int nvgpu_clk_arb_release_completion_dev(struct inode *inode,
1625 struct file *filp)
1626{
1627 struct nvgpu_clk_dev *dev = filp->private_data;
1628 struct nvgpu_clk_session *session = dev->session;
1629
1630
1631 gk20a_dbg_fn("");
1632
1633 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
1634 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1635 return 0;
1636}
1637
1638static int nvgpu_clk_arb_release_event_dev(struct inode *inode,
1639 struct file *filp)
1640{
1641 struct nvgpu_clk_dev *dev = filp->private_data;
1642 struct nvgpu_clk_session *session = dev->session;
1643 struct nvgpu_clk_arb *arb;
1644
1645 arb = session->g->clk_arb;
1646
1647 gk20a_dbg_fn("");
1648
1649 if (arb) {
1650 nvgpu_spinlock_acquire(&arb->users_lock);
1651 list_del_rcu(&dev->link);
1652 nvgpu_spinlock_release(&arb->users_lock);
1653 nvgpu_clk_notification_queue_free(arb->g, &dev->queue);
1654 }
1655
1656 synchronize_rcu();
1657 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
1658 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1659
1660 return 0;
1661}
1662
1663int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session,
1664 int request_fd, u32 api_domain, u16 target_mhz)
1665{
1666 struct nvgpu_clk_dev *dev;
1667 struct fd fd;
1668 int err = 0;
1669
1670 gk20a_dbg_fn("domain=0x%08x target_mhz=%u", api_domain, target_mhz);
1671
1672 fd = fdget(request_fd);
1673 if (!fd.file)
1674 return -EINVAL;
1675
1676 if (fd.file->f_op != &completion_dev_ops) {
1677 err = -EINVAL;
1678 goto fdput_fd;
1679 }
1680
1681 dev = fd.file->private_data;
1682 if (!dev || dev->session != session) {
1683 err = -EINVAL;
1684 goto fdput_fd;
1685 }
1686
1687 switch (api_domain) {
1688 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1689 dev->mclk_target_mhz = target_mhz;
1690 break;
1691
1692 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1693 dev->gpc2clk_target_mhz = target_mhz * 2ULL;
1694 break;
1695
1696 default:
1697 err = -EINVAL;
1698 }
1699
1700fdput_fd:
1701 fdput(fd);
1702 return err;
1703}
1704
1705int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session,
1706 u32 api_domain, u16 *freq_mhz)
1707{
1708 int err = 0;
1709 struct nvgpu_clk_arb_target *target;
1710
1711 do {
1712 target = NV_ACCESS_ONCE(session->target);
1713 /* no reordering of this pointer */
1714 nvgpu_smp_rmb();
1715
1716 switch (api_domain) {
1717 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1718 *freq_mhz = target->mclk;
1719 break;
1720
1721 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1722 *freq_mhz = target->gpc2clk / 2ULL;
1723 break;
1724
1725 default:
1726 *freq_mhz = 0;
1727 err = -EINVAL;
1728 }
1729 } while (target != NV_ACCESS_ONCE(session->target));
1730 return err;
1731}
1732
1733int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g,
1734 u32 api_domain, u16 *freq_mhz)
1735{
1736 struct nvgpu_clk_arb *arb = g->clk_arb;
1737 int err = 0;
1738 struct nvgpu_clk_arb_target *actual;
1739
1740 do {
1741 actual = NV_ACCESS_ONCE(arb->actual);
1742 /* no reordering of this pointer */
1743 nvgpu_smp_rmb();
1744
1745 switch (api_domain) {
1746 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1747 *freq_mhz = actual->mclk;
1748 break;
1749
1750 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1751 *freq_mhz = actual->gpc2clk / 2ULL;
1752 break;
1753
1754 default:
1755 *freq_mhz = 0;
1756 err = -EINVAL;
1757 }
1758 } while (actual != NV_ACCESS_ONCE(arb->actual));
1759 return err;
1760}
1761
1762int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g,
1763 u32 api_domain, u16 *freq_mhz)
1764{
1765 switch (api_domain) {
1766 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1767 *freq_mhz = g->ops.clk.measure_freq(g, CTRL_CLK_DOMAIN_MCLK) /
1768 1000000ULL;
1769 return 0;
1770
1771 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1772 *freq_mhz = g->ops.clk.measure_freq(g,
1773 CTRL_CLK_DOMAIN_GPC2CLK) / 2000000ULL;
1774 return 0;
1775
1776 default:
1777 return -EINVAL;
1778 }
1779}
1780
1781int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
1782 u16 *min_mhz, u16 *max_mhz)
1783{
1784 int ret;
1785
1786 switch (api_domain) {
1787 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1788 ret = g->ops.clk_arb.get_arbiter_clk_range(g,
1789 CTRL_CLK_DOMAIN_MCLK, min_mhz, max_mhz);
1790 return ret;
1791
1792 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1793 ret = g->ops.clk_arb.get_arbiter_clk_range(g,
1794 CTRL_CLK_DOMAIN_GPC2CLK, min_mhz, max_mhz);
1795 if (!ret) {
1796 *min_mhz /= 2;
1797 *max_mhz /= 2;
1798 }
1799 return ret;
1800
1801 default:
1802 return -EINVAL;
1803 }
1804}
1805
1806u32 nvgpu_clk_arb_get_arbiter_clk_domains(struct gk20a *g)
1807{
1808 u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g);
1809 u32 api_domains = 0;
1810
1811 if (clk_domains & CTRL_CLK_DOMAIN_GPC2CLK)
1812 api_domains |= BIT(NVGPU_GPU_CLK_DOMAIN_GPCCLK);
1813
1814 if (clk_domains & CTRL_CLK_DOMAIN_MCLK)
1815 api_domains |= BIT(NVGPU_GPU_CLK_DOMAIN_MCLK);
1816
1817 return api_domains;
1818}
1819
1820bool nvgpu_clk_arb_is_valid_domain(struct gk20a *g, u32 api_domain)
1821{
1822 u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g);
1823
1824 switch (api_domain) {
1825 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1826 return ((clk_domains & CTRL_CLK_DOMAIN_MCLK) != 0);
1827
1828 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1829 return ((clk_domains & CTRL_CLK_DOMAIN_GPC2CLK) != 0);
1830
1831 default:
1832 return false;
1833 }
1834}
1835
1836int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g,
1837 u32 api_domain, u32 *max_points, u16 *fpoints)
1838{
1839 int err;
1840 u32 i;
1841
1842 switch (api_domain) {
1843 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1844 err = clk_domain_get_f_points(g, CTRL_CLK_DOMAIN_GPC2CLK,
1845 max_points, fpoints);
1846 if (err || !fpoints)
1847 return err;
1848 for (i = 0; i < *max_points; i++)
1849 fpoints[i] /= 2;
1850 return 0;
1851 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1852 return clk_domain_get_f_points(g, CTRL_CLK_DOMAIN_MCLK,
1853 max_points, fpoints);
1854 default:
1855 return -EINVAL;
1856 }
1857}
1858
1859static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb,
1860 u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk,
1861 u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram)
1862{
1863 u16 gpc2clk_target, mclk_target;
1864 u32 gpc2clk_voltuv, gpc2clk_voltuv_sram;
1865 u32 mclk_voltuv, mclk_voltuv_sram;
1866 u32 pstate = VF_POINT_INVALID_PSTATE;
1867 struct nvgpu_clk_vf_table *table;
1868 u32 index, index_mclk;
1869 struct nvgpu_clk_vf_point *mclk_vf = NULL;
1870
1871 do {
1872 gpc2clk_target = *gpc2clk;
1873 mclk_target = *mclk;
1874 gpc2clk_voltuv = 0;
1875 gpc2clk_voltuv_sram = 0;
1876 mclk_voltuv = 0;
1877 mclk_voltuv_sram = 0;
1878
1879 table = NV_ACCESS_ONCE(arb->current_vf_table);
1880 /* pointer to table can be updated by callback */
1881 nvgpu_smp_rmb();
1882
1883 if (!table)
1884 continue;
1885 if ((!table->gpc2clk_num_points) || (!table->mclk_num_points)) {
1886 nvgpu_err(arb->g, "found empty table");
1887 goto find_exit;
1888 }
1889 /* First we check MCLK to find out which PSTATE we are
1890 * are requesting, and from there try to find the minimum
1891 * GPC2CLK on the same PSTATE that satisfies the request.
1892 * If no GPC2CLK can be found, then we need to up the PSTATE
1893 */
1894
1895recalculate_vf_point:
1896 for (index = 0; index < table->mclk_num_points; index++) {
1897 if (table->mclk_points[index].mem_mhz >= mclk_target) {
1898 mclk_vf = &table->mclk_points[index];
1899 break;
1900 }
1901 }
1902 if (index == table->mclk_num_points) {
1903 mclk_vf = &table->mclk_points[index-1];
1904 index = table->mclk_num_points - 1;
1905 }
1906 index_mclk = index;
1907
1908 /* round up the freq requests */
1909 for (index = 0; index < table->gpc2clk_num_points; index++) {
1910 pstate = VF_POINT_COMMON_PSTATE(
1911 &table->gpc2clk_points[index], mclk_vf);
1912
1913 if ((table->gpc2clk_points[index].gpc_mhz >=
1914 gpc2clk_target) &&
1915 (pstate != VF_POINT_INVALID_PSTATE)) {
1916 gpc2clk_target =
1917 table->gpc2clk_points[index].gpc_mhz;
1918 *sys2clk =
1919 table->gpc2clk_points[index].sys_mhz;
1920 *xbar2clk =
1921 table->gpc2clk_points[index].xbar_mhz;
1922
1923 gpc2clk_voltuv =
1924 table->gpc2clk_points[index].uvolt;
1925 gpc2clk_voltuv_sram =
1926 table->gpc2clk_points[index].uvolt_sram;
1927 break;
1928 }
1929 }
1930
1931 if (index == table->gpc2clk_num_points) {
1932 pstate = VF_POINT_COMMON_PSTATE(
1933 &table->gpc2clk_points[index-1], mclk_vf);
1934 if (pstate != VF_POINT_INVALID_PSTATE) {
1935 gpc2clk_target =
1936 table->gpc2clk_points[index-1].gpc_mhz;
1937 *sys2clk =
1938 table->gpc2clk_points[index-1].sys_mhz;
1939 *xbar2clk =
1940 table->gpc2clk_points[index-1].xbar_mhz;
1941
1942 gpc2clk_voltuv =
1943 table->gpc2clk_points[index-1].uvolt;
1944 gpc2clk_voltuv_sram =
1945 table->gpc2clk_points[index-1].
1946 uvolt_sram;
1947 } else if (index_mclk >= table->mclk_num_points - 1) {
1948 /* There is no available combination of MCLK
1949 * and GPC2CLK, we need to fail this
1950 */
1951 gpc2clk_target = 0;
1952 mclk_target = 0;
1953 pstate = VF_POINT_INVALID_PSTATE;
1954 goto find_exit;
1955 } else {
1956 /* recalculate with higher PSTATE */
1957 gpc2clk_target = *gpc2clk;
1958 mclk_target = table->mclk_points[index_mclk+1].
1959 mem_mhz;
1960 goto recalculate_vf_point;
1961 }
1962 }
1963
1964 mclk_target = mclk_vf->mem_mhz;
1965 mclk_voltuv = mclk_vf->uvolt;
1966 mclk_voltuv_sram = mclk_vf->uvolt_sram;
1967
1968 } while (!table ||
1969 (NV_ACCESS_ONCE(arb->current_vf_table) != table));
1970
1971find_exit:
1972 *voltuv = gpc2clk_voltuv > mclk_voltuv ? gpc2clk_voltuv : mclk_voltuv;
1973 *voltuv_sram = gpc2clk_voltuv_sram > mclk_voltuv_sram ?
1974 gpc2clk_voltuv_sram : mclk_voltuv_sram;
1975 /* noise unaware vmin */
1976 *nuvmin = mclk_voltuv;
1977 *nuvmin_sram = mclk_voltuv_sram;
1978 *gpc2clk = gpc2clk_target < *gpc2clk ? gpc2clk_target : *gpc2clk;
1979 *mclk = mclk_target;
1980 return pstate;
1981}
1982
1983/* This function is inherently unsafe to call while arbiter is running
1984 * arbiter must be blocked before calling this function
1985 */
1986int nvgpu_clk_arb_get_current_pstate(struct gk20a *g)
1987{
1988 return NV_ACCESS_ONCE(g->clk_arb->actual->pstate);
1989}
1990
1991static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target,
1992 u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv,
1993 u32 voltuv_sram)
1994{
1995 struct set_fll_clk fllclk;
1996 struct nvgpu_clk_arb *arb = g->clk_arb;
1997 int status;
1998
1999 fllclk.gpc2clkmhz = gpc2clk_target;
2000 fllclk.sys2clkmhz = sys2clk_target;
2001 fllclk.xbar2clkmhz = xbar2clk_target;
2002
2003 fllclk.voltuv = voltuv;
2004
2005 /* if voltage ascends we do:
2006 * (1) FLL change
2007 * (2) Voltage change
2008 * (3) MCLK change
2009 * If it goes down
2010 * (1) MCLK change
2011 * (2) Voltage change
2012 * (3) FLL change
2013 */
2014
2015 /* descending */
2016 if (voltuv < arb->voltuv_actual) {
2017 status = g->ops.clk.mclk_change(g, mclk_target);
2018 if (status < 0)
2019 return status;
2020
2021 status = volt_set_voltage(g, voltuv, voltuv_sram);
2022 if (status < 0)
2023 return status;
2024
2025 status = clk_set_fll_clks(g, &fllclk);
2026 if (status < 0)
2027 return status;
2028 } else {
2029 status = clk_set_fll_clks(g, &fllclk);
2030 if (status < 0)
2031 return status;
2032
2033 status = volt_set_voltage(g, voltuv, voltuv_sram);
2034 if (status < 0)
2035 return status;
2036
2037 status = g->ops.clk.mclk_change(g, mclk_target);
2038 if (status < 0)
2039 return status;
2040 }
2041
2042 return 0;
2043}
2044
2045void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock)
2046{
2047 struct nvgpu_clk_arb *arb = g->clk_arb;
2048
2049 if (lock)
2050 nvgpu_mutex_acquire(&arb->pstate_lock);
2051 else
2052 nvgpu_mutex_release(&arb->pstate_lock);
2053}
2054
2055#ifdef CONFIG_DEBUG_FS
2056static int nvgpu_clk_arb_stats_show(struct seq_file *s, void *unused)
2057{
2058 struct gk20a *g = s->private;
2059 struct nvgpu_clk_arb *arb = g->clk_arb;
2060 struct nvgpu_clk_arb_debug *debug;
2061
2062 u64 num;
2063 s64 tmp, avg, std, max, min;
2064
2065 debug = NV_ACCESS_ONCE(arb->debug);
2066 /* Make copy of structure and ensure no reordering */
2067 nvgpu_smp_rmb();
2068 if (!debug)
2069 return -EINVAL;
2070
2071 std = debug->switch_std;
2072 avg = debug->switch_avg;
2073 max = debug->switch_max;
2074 min = debug->switch_min;
2075 num = debug->switch_num;
2076
2077 tmp = std;
2078 do_div(tmp, num);
2079 seq_printf(s, "Number of transitions: %lld\n",
2080 num);
2081 seq_printf(s, "max / min : %lld / %lld usec\n",
2082 max, min);
2083 seq_printf(s, "avg / std : %lld / %ld usec\n",
2084 avg, int_sqrt(tmp));
2085
2086 return 0;
2087}
2088
2089static int nvgpu_clk_arb_stats_open(struct inode *inode, struct file *file)
2090{
2091 return single_open(file, nvgpu_clk_arb_stats_show, inode->i_private);
2092}
2093
2094static const struct file_operations nvgpu_clk_arb_stats_fops = {
2095 .open = nvgpu_clk_arb_stats_open,
2096 .read = seq_read,
2097 .llseek = seq_lseek,
2098 .release = single_release,
2099};
2100
2101
2102static int nvgpu_clk_arb_debugfs_init(struct gk20a *g)
2103{
2104 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
2105 struct dentry *gpu_root = l->debugfs;
2106 struct dentry *d;
2107
2108 gk20a_dbg(gpu_dbg_info, "g=%p", g);
2109
2110 d = debugfs_create_file(
2111 "arb_stats",
2112 S_IRUGO,
2113 gpu_root,
2114 g,
2115 &nvgpu_clk_arb_stats_fops);
2116 if (!d)
2117 return -ENOMEM;
2118
2119 return 0;
2120}
2121#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-05 02:29:36 -0400