summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/linux/clk_arb.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/clk_arb.c')
-rw-r--r--drivers/gpu/nvgpu/common/linux/clk_arb.c2122
1 files changed, 2122 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/clk_arb.c b/drivers/gpu/nvgpu/common/linux/clk_arb.c
new file mode 100644
index 00000000..ef89f980
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/clk_arb.c
@@ -0,0 +1,2122 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23
24#include <linux/cdev.h>
25#include <linux/file.h>
26#include <linux/anon_inodes.h>
27#include <linux/rculist.h>
28#include <linux/llist.h>
29#include <linux/uaccess.h>
30#include <linux/poll.h>
31#ifdef CONFIG_DEBUG_FS
32#include <linux/debugfs.h>
33#endif
34#include <uapi/linux/nvgpu.h>
35
36#include <nvgpu/bitops.h>
37#include <nvgpu/lock.h>
38#include <nvgpu/kmem.h>
39#include <nvgpu/atomic.h>
40#include <nvgpu/bug.h>
41#include <nvgpu/kref.h>
42#include <nvgpu/log.h>
43#include <nvgpu/barrier.h>
44#include <nvgpu/cond.h>
45#include <nvgpu/clk_arb.h>
46
47#include "gk20a/gk20a.h"
48#include "clk/clk.h"
49#include "pstate/pstate.h"
50#include "lpwr/lpwr.h"
51#include "volt/volt.h"
52
53#ifdef CONFIG_DEBUG_FS
54#include "common/linux/os_linux.h"
55#endif
56
57#define MAX_F_POINTS 256
58#define DEFAULT_EVENT_NUMBER 32
59
60struct nvgpu_clk_dev;
61struct nvgpu_clk_arb_target;
62struct nvgpu_clk_notification_queue;
63
64#ifdef CONFIG_DEBUG_FS
65static int nvgpu_clk_arb_debugfs_init(struct gk20a *g);
66#endif
67
68static int nvgpu_clk_arb_release_event_dev(struct inode *inode,
69 struct file *filp);
70static int nvgpu_clk_arb_release_completion_dev(struct inode *inode,
71 struct file *filp);
72static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait);
73static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf,
74 size_t size, loff_t *off);
75
76static long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd,
77 unsigned long arg);
78
79static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work);
80static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work);
81static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb);
82static void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount);
83static void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount);
84static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target,
85 u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv,
86 u32 voltuv_sram);
87static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb,
88 u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk,
89 u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram);
90static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev,
91 struct nvgpu_clk_arb_target *target,
92 u32 alarm_mask);
93static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm);
94static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm);
95
96static void nvgpu_clk_arb_queue_notification(struct gk20a *g,
97 struct nvgpu_clk_notification_queue *queue,
98 u32 alarm_mask);
99static int nvgpu_clk_notification_queue_alloc(struct gk20a *g,
100 struct nvgpu_clk_notification_queue *queue,
101 size_t events_number);
102
103static void nvgpu_clk_notification_queue_free(struct gk20a *g,
104 struct nvgpu_clk_notification_queue *queue);
105
106#define VF_POINT_INVALID_PSTATE ~0U
107#define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b)))
108#define VF_POINT_GET_PSTATE(a) (((a)->pstates) ?\
109 __fls((a)->pstates) :\
110 VF_POINT_INVALID_PSTATE)
111#define VF_POINT_COMMON_PSTATE(a, b) (((a)->pstates & (b)->pstates) ?\
112 __fls((a)->pstates & (b)->pstates) :\
113 VF_POINT_INVALID_PSTATE)
114
115/* Local Alarms */
116#define EVENT(alarm) (0x1UL << NVGPU_GPU_EVENT_##alarm)
117
118#define LOCAL_ALARM_MASK (EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE) | \
119 EVENT(VF_UPDATE))
120
121#define _WRAPGTEQ(a, b) ((a-b) > 0)
122
123struct nvgpu_clk_notification {
124 u32 notification;
125 u64 timestamp;
126};
127
128struct nvgpu_clk_notification_queue {
129 u32 size;
130 nvgpu_atomic_t head;
131 nvgpu_atomic_t tail;
132 struct nvgpu_clk_notification *notifications;
133};
134
135struct nvgpu_clk_vf_point {
136 u16 pstates;
137 union {
138 struct {
139 u16 gpc_mhz;
140 u16 sys_mhz;
141 u16 xbar_mhz;
142 };
143 u16 mem_mhz;
144 };
145 u32 uvolt;
146 u32 uvolt_sram;
147};
148
149struct nvgpu_clk_vf_table {
150 u32 mclk_num_points;
151 struct nvgpu_clk_vf_point *mclk_points;
152 u32 gpc2clk_num_points;
153 struct nvgpu_clk_vf_point *gpc2clk_points;
154};
155#ifdef CONFIG_DEBUG_FS
156struct nvgpu_clk_arb_debug {
157 s64 switch_max;
158 s64 switch_min;
159 u64 switch_num;
160 s64 switch_avg;
161 s64 switch_std;
162};
163#endif
164
165struct nvgpu_clk_arb_target {
166 u16 mclk;
167 u16 gpc2clk;
168 u32 pstate;
169};
170
171struct nvgpu_clk_arb {
172 struct nvgpu_spinlock sessions_lock;
173 struct nvgpu_spinlock users_lock;
174
175 struct nvgpu_mutex pstate_lock;
176 struct list_head users;
177 struct list_head sessions;
178 struct llist_head requests;
179
180 struct gk20a *g;
181 int status;
182
183 struct nvgpu_clk_arb_target actual_pool[2];
184 struct nvgpu_clk_arb_target *actual;
185
186 u16 gpc2clk_default_mhz;
187 u16 mclk_default_mhz;
188 u32 voltuv_actual;
189
190 u16 gpc2clk_min, gpc2clk_max;
191 u16 mclk_min, mclk_max;
192
193 struct work_struct update_fn_work;
194 struct workqueue_struct *update_work_queue;
195 struct work_struct vf_table_fn_work;
196 struct workqueue_struct *vf_table_work_queue;
197
198 struct nvgpu_cond request_wq;
199
200 struct nvgpu_clk_vf_table *current_vf_table;
201 struct nvgpu_clk_vf_table vf_table_pool[2];
202 u32 vf_table_index;
203
204 u16 *mclk_f_points;
205 nvgpu_atomic_t req_nr;
206
207 u32 mclk_f_numpoints;
208 u16 *gpc2clk_f_points;
209 u32 gpc2clk_f_numpoints;
210
211 nvgpu_atomic64_t alarm_mask;
212 struct nvgpu_clk_notification_queue notification_queue;
213
214#ifdef CONFIG_DEBUG_FS
215 struct nvgpu_clk_arb_debug debug_pool[2];
216 struct nvgpu_clk_arb_debug *debug;
217 bool debugfs_set;
218#endif
219};
220
221struct nvgpu_clk_dev {
222 struct nvgpu_clk_session *session;
223 union {
224 struct list_head link;
225 struct llist_node node;
226 };
227 struct nvgpu_cond readout_wq;
228 nvgpu_atomic_t poll_mask;
229 u16 gpc2clk_target_mhz;
230 u16 mclk_target_mhz;
231 u32 alarms_reported;
232 nvgpu_atomic_t enabled_mask;
233 struct nvgpu_clk_notification_queue queue;
234 u32 arb_queue_head;
235 struct nvgpu_ref refcount;
236};
237
238struct nvgpu_clk_session {
239 bool zombie;
240 struct gk20a *g;
241 struct nvgpu_ref refcount;
242 struct list_head link;
243 struct llist_head targets;
244
245 struct nvgpu_clk_arb_target target_pool[2];
246 struct nvgpu_clk_arb_target *target;
247};
248
249static const struct file_operations completion_dev_ops = {
250 .owner = THIS_MODULE,
251 .release = nvgpu_clk_arb_release_completion_dev,
252 .poll = nvgpu_clk_arb_poll_dev,
253};
254
255static const struct file_operations event_dev_ops = {
256 .owner = THIS_MODULE,
257 .release = nvgpu_clk_arb_release_event_dev,
258 .poll = nvgpu_clk_arb_poll_dev,
259 .read = nvgpu_clk_arb_read_event_dev,
260#ifdef CONFIG_COMPAT
261 .compat_ioctl = nvgpu_clk_arb_ioctl_event_dev,
262#endif
263 .unlocked_ioctl = nvgpu_clk_arb_ioctl_event_dev,
264};
265
266static int nvgpu_clk_notification_queue_alloc(struct gk20a *g,
267 struct nvgpu_clk_notification_queue *queue,
268 size_t events_number) {
269 queue->notifications = nvgpu_kcalloc(g, events_number,
270 sizeof(struct nvgpu_clk_notification));
271 if (!queue->notifications)
272 return -ENOMEM;
273 queue->size = events_number;
274
275 nvgpu_atomic_set(&queue->head, 0);
276 nvgpu_atomic_set(&queue->tail, 0);
277
278 return 0;
279}
280
281static void nvgpu_clk_notification_queue_free(struct gk20a *g,
282 struct nvgpu_clk_notification_queue *queue) {
283 nvgpu_kfree(g, queue->notifications);
284 queue->size = 0;
285 nvgpu_atomic_set(&queue->head, 0);
286 nvgpu_atomic_set(&queue->tail, 0);
287}
288
289int nvgpu_clk_arb_init_arbiter(struct gk20a *g)
290{
291 struct nvgpu_clk_arb *arb;
292 u16 default_mhz;
293 int err;
294 int index;
295 struct nvgpu_clk_vf_table *table;
296
297 gk20a_dbg_fn("");
298
299 if (!g->ops.clk_arb.get_arbiter_clk_domains)
300 return 0;
301
302 arb = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_arb));
303 if (!arb)
304 return -ENOMEM;
305
306 err = nvgpu_mutex_init(&arb->pstate_lock);
307 if (err)
308 goto mutex_fail;
309 nvgpu_spinlock_init(&arb->sessions_lock);
310 nvgpu_spinlock_init(&arb->users_lock);
311
312 arb->mclk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16));
313 if (!arb->mclk_f_points) {
314 err = -ENOMEM;
315 goto init_fail;
316 }
317
318 arb->gpc2clk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16));
319 if (!arb->gpc2clk_f_points) {
320 err = -ENOMEM;
321 goto init_fail;
322 }
323
324 for (index = 0; index < 2; index++) {
325 table = &arb->vf_table_pool[index];
326 table->gpc2clk_num_points = MAX_F_POINTS;
327 table->mclk_num_points = MAX_F_POINTS;
328
329 table->gpc2clk_points = nvgpu_kcalloc(g, MAX_F_POINTS,
330 sizeof(struct nvgpu_clk_vf_point));
331 if (!table->gpc2clk_points) {
332 err = -ENOMEM;
333 goto init_fail;
334 }
335
336
337 table->mclk_points = nvgpu_kcalloc(g, MAX_F_POINTS,
338 sizeof(struct nvgpu_clk_vf_point));
339 if (!table->mclk_points) {
340 err = -ENOMEM;
341 goto init_fail;
342 }
343 }
344
345 g->clk_arb = arb;
346 arb->g = g;
347
348 err = g->ops.clk_arb.get_arbiter_clk_default(g,
349 CTRL_CLK_DOMAIN_MCLK, &default_mhz);
350 if (err < 0) {
351 err = -EINVAL;
352 goto init_fail;
353 }
354
355 arb->mclk_default_mhz = default_mhz;
356
357 err = g->ops.clk_arb.get_arbiter_clk_default(g,
358 CTRL_CLK_DOMAIN_GPC2CLK, &default_mhz);
359 if (err < 0) {
360 err = -EINVAL;
361 goto init_fail;
362 }
363
364 arb->gpc2clk_default_mhz = default_mhz;
365
366 arb->actual = &arb->actual_pool[0];
367
368 nvgpu_atomic_set(&arb->req_nr, 0);
369
370 nvgpu_atomic64_set(&arb->alarm_mask, 0);
371 err = nvgpu_clk_notification_queue_alloc(g, &arb->notification_queue,
372 DEFAULT_EVENT_NUMBER);
373 if (err < 0)
374 goto init_fail;
375
376 INIT_LIST_HEAD_RCU(&arb->users);
377 INIT_LIST_HEAD_RCU(&arb->sessions);
378 init_llist_head(&arb->requests);
379
380 nvgpu_cond_init(&arb->request_wq);
381 arb->vf_table_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1,
382 "vf_table_update");
383 arb->update_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1,
384 "arbiter_update");
385
386
387 INIT_WORK(&arb->vf_table_fn_work, nvgpu_clk_arb_run_vf_table_cb);
388
389 INIT_WORK(&arb->update_fn_work, nvgpu_clk_arb_run_arbiter_cb);
390
391#ifdef CONFIG_DEBUG_FS
392 arb->debug = &arb->debug_pool[0];
393
394 if (!arb->debugfs_set) {
395 if (nvgpu_clk_arb_debugfs_init(g))
396 arb->debugfs_set = true;
397 }
398#endif
399 err = clk_vf_point_cache(g);
400 if (err < 0)
401 goto init_fail;
402
403 err = nvgpu_clk_arb_update_vf_table(arb);
404 if (err < 0)
405 goto init_fail;
406 do {
407 /* Check that first run is completed */
408 nvgpu_smp_mb();
409 NVGPU_COND_WAIT_INTERRUPTIBLE(&arb->request_wq,
410 nvgpu_atomic_read(&arb->req_nr), 0);
411 } while (!nvgpu_atomic_read(&arb->req_nr));
412
413
414 return arb->status;
415
416init_fail:
417 nvgpu_kfree(g, arb->gpc2clk_f_points);
418 nvgpu_kfree(g, arb->mclk_f_points);
419
420 for (index = 0; index < 2; index++) {
421 nvgpu_kfree(g, arb->vf_table_pool[index].gpc2clk_points);
422 nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points);
423 }
424
425 nvgpu_mutex_destroy(&arb->pstate_lock);
426
427mutex_fail:
428 nvgpu_kfree(g, arb);
429
430 return err;
431}
432
433void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm)
434{
435 struct nvgpu_clk_arb *arb = g->clk_arb;
436
437 nvgpu_clk_arb_set_global_alarm(g, alarm);
438 if (arb->update_work_queue)
439 queue_work(arb->update_work_queue, &arb->update_fn_work);
440}
441
442static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm)
443{
444 struct nvgpu_clk_arb *arb = g->clk_arb;
445
446 u64 current_mask;
447 u32 refcnt;
448 u32 alarm_mask;
449 u64 new_mask;
450
451 do {
452 current_mask = nvgpu_atomic64_read(&arb->alarm_mask);
453 /* atomic operations are strong so they do not need masks */
454
455 refcnt = ((u32) (current_mask >> 32)) + 1;
456 alarm_mask = (u32) (current_mask & ~alarm);
457 new_mask = ((u64) refcnt << 32) | alarm_mask;
458
459 } while (unlikely(current_mask !=
460 (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask,
461 current_mask, new_mask)));
462}
463
464static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm)
465{
466 struct nvgpu_clk_arb *arb = g->clk_arb;
467
468 u64 current_mask;
469 u32 refcnt;
470 u32 alarm_mask;
471 u64 new_mask;
472
473 do {
474 current_mask = nvgpu_atomic64_read(&arb->alarm_mask);
475 /* atomic operations are strong so they do not need masks */
476
477 refcnt = ((u32) (current_mask >> 32)) + 1;
478 alarm_mask = (u32) (current_mask & ~0) | alarm;
479 new_mask = ((u64) refcnt << 32) | alarm_mask;
480
481 } while (unlikely(current_mask !=
482 (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask,
483 current_mask, new_mask)));
484
485 nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm);
486
487}
488
489void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g)
490{
491 struct nvgpu_clk_arb *arb = g->clk_arb;
492 int index;
493
494 if (arb) {
495 cancel_work_sync(&arb->vf_table_fn_work);
496 destroy_workqueue(arb->vf_table_work_queue);
497 arb->vf_table_work_queue = NULL;
498
499 cancel_work_sync(&arb->update_fn_work);
500 destroy_workqueue(arb->update_work_queue);
501 arb->update_work_queue = NULL;
502
503 nvgpu_kfree(g, arb->gpc2clk_f_points);
504 nvgpu_kfree(g, arb->mclk_f_points);
505
506 for (index = 0; index < 2; index++) {
507 nvgpu_kfree(g,
508 arb->vf_table_pool[index].gpc2clk_points);
509 nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points);
510 }
511 nvgpu_mutex_destroy(&g->clk_arb->pstate_lock);
512 nvgpu_kfree(g, g->clk_arb);
513 g->clk_arb = NULL;
514 }
515}
516
517static int nvgpu_clk_arb_install_fd(struct gk20a *g,
518 struct nvgpu_clk_session *session,
519 const struct file_operations *fops,
520 struct nvgpu_clk_dev **_dev)
521{
522 struct file *file;
523 int fd;
524 int err;
525 int status;
526 char name[64];
527 struct nvgpu_clk_dev *dev;
528
529 gk20a_dbg_fn("");
530
531 dev = nvgpu_kzalloc(g, sizeof(*dev));
532 if (!dev)
533 return -ENOMEM;
534
535 status = nvgpu_clk_notification_queue_alloc(g, &dev->queue,
536 DEFAULT_EVENT_NUMBER);
537 if (status < 0) {
538 err = status;
539 goto fail;
540 }
541
542 fd = get_unused_fd_flags(O_RDWR);
543 if (fd < 0) {
544 err = fd;
545 goto fail;
546 }
547
548 snprintf(name, sizeof(name), "%s-clk-fd%d", g->name, fd);
549 file = anon_inode_getfile(name, fops, dev, O_RDWR);
550 if (IS_ERR(file)) {
551 err = PTR_ERR(file);
552 goto fail_fd;
553 }
554
555 fd_install(fd, file);
556
557 nvgpu_cond_init(&dev->readout_wq);
558
559 nvgpu_atomic_set(&dev->poll_mask, 0);
560
561 dev->session = session;
562 nvgpu_ref_init(&dev->refcount);
563
564 nvgpu_ref_get(&session->refcount);
565
566 *_dev = dev;
567
568 return fd;
569
570fail_fd:
571 put_unused_fd(fd);
572fail:
573 nvgpu_kfree(g, dev);
574
575 return err;
576}
577
578int nvgpu_clk_arb_init_session(struct gk20a *g,
579 struct nvgpu_clk_session **_session)
580{
581 struct nvgpu_clk_arb *arb = g->clk_arb;
582 struct nvgpu_clk_session *session = *(_session);
583
584 gk20a_dbg_fn("");
585
586 if (!g->ops.clk_arb.get_arbiter_clk_domains)
587 return 0;
588
589 session = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_session));
590 if (!session)
591 return -ENOMEM;
592 session->g = g;
593
594 nvgpu_ref_init(&session->refcount);
595
596 session->zombie = false;
597 session->target_pool[0].pstate = CTRL_PERF_PSTATE_P8;
598 /* make sure that the initialization of the pool is visible
599 * before the update
600 */
601 nvgpu_smp_wmb();
602 session->target = &session->target_pool[0];
603
604 init_llist_head(&session->targets);
605
606 nvgpu_spinlock_acquire(&arb->sessions_lock);
607 list_add_tail_rcu(&session->link, &arb->sessions);
608 nvgpu_spinlock_release(&arb->sessions_lock);
609
610 *_session = session;
611
612 return 0;
613}
614
615static void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount)
616{
617 struct nvgpu_clk_dev *dev = container_of(refcount,
618 struct nvgpu_clk_dev, refcount);
619 struct nvgpu_clk_session *session = dev->session;
620
621 nvgpu_kfree(session->g, dev);
622}
623
624static void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount)
625{
626 struct nvgpu_clk_session *session = container_of(refcount,
627 struct nvgpu_clk_session, refcount);
628 struct nvgpu_clk_arb *arb = session->g->clk_arb;
629 struct gk20a *g = session->g;
630 struct nvgpu_clk_dev *dev, *tmp;
631 struct llist_node *head;
632
633 gk20a_dbg_fn("");
634
635 if (arb) {
636 nvgpu_spinlock_acquire(&arb->sessions_lock);
637 list_del_rcu(&session->link);
638 nvgpu_spinlock_release(&arb->sessions_lock);
639 }
640
641 head = llist_del_all(&session->targets);
642 llist_for_each_entry_safe(dev, tmp, head, node) {
643 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
644 }
645 synchronize_rcu();
646 nvgpu_kfree(g, session);
647}
648
649void nvgpu_clk_arb_release_session(struct gk20a *g,
650 struct nvgpu_clk_session *session)
651{
652 struct nvgpu_clk_arb *arb = g->clk_arb;
653
654 gk20a_dbg_fn("");
655
656 session->zombie = true;
657 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
658 if (arb && arb->update_work_queue)
659 queue_work(arb->update_work_queue, &arb->update_fn_work);
660}
661
662int nvgpu_clk_arb_install_event_fd(struct gk20a *g,
663 struct nvgpu_clk_session *session, int *event_fd, u32 alarm_mask)
664{
665 struct nvgpu_clk_arb *arb = g->clk_arb;
666 struct nvgpu_clk_dev *dev;
667 int fd;
668
669 gk20a_dbg_fn("");
670
671 fd = nvgpu_clk_arb_install_fd(g, session, &event_dev_ops, &dev);
672 if (fd < 0)
673 return fd;
674
675 /* TODO: alarm mask needs to be set to default value to prevent
676 * failures of legacy tests. This will be removed when sanity is
677 * updated
678 */
679 if (alarm_mask)
680 nvgpu_atomic_set(&dev->enabled_mask, alarm_mask);
681 else
682 nvgpu_atomic_set(&dev->enabled_mask, EVENT(VF_UPDATE));
683
684 dev->arb_queue_head = nvgpu_atomic_read(&arb->notification_queue.head);
685
686 nvgpu_spinlock_acquire(&arb->users_lock);
687 list_add_tail_rcu(&dev->link, &arb->users);
688 nvgpu_spinlock_release(&arb->users_lock);
689
690 *event_fd = fd;
691
692 return 0;
693}
694
695int nvgpu_clk_arb_install_request_fd(struct gk20a *g,
696 struct nvgpu_clk_session *session, int *request_fd)
697{
698 struct nvgpu_clk_dev *dev;
699 int fd;
700
701 gk20a_dbg_fn("");
702
703 fd = nvgpu_clk_arb_install_fd(g, session, &completion_dev_ops, &dev);
704 if (fd < 0)
705 return fd;
706
707 *request_fd = fd;
708
709 return 0;
710}
711
712static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb)
713{
714 struct gk20a *g = arb->g;
715 struct nvgpu_clk_vf_table *table;
716
717 u32 i, j;
718 int status = -EINVAL;
719 u32 gpc2clk_voltuv = 0, mclk_voltuv = 0;
720 u32 gpc2clk_voltuv_sram = 0, mclk_voltuv_sram = 0;
721 u16 clk_cur;
722 u32 num_points;
723
724 struct clk_set_info *p5_info, *p0_info;
725
726
727 table = NV_ACCESS_ONCE(arb->current_vf_table);
728 /* make flag visible when all data has resolved in the tables */
729 nvgpu_smp_rmb();
730
731 table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] :
732 &arb->vf_table_pool[0];
733
734 /* Get allowed memory ranges */
735 if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_GPC2CLK,
736 &arb->gpc2clk_min,
737 &arb->gpc2clk_max) < 0) {
738 nvgpu_err(g, "failed to fetch GPC2CLK range");
739 goto exit_vf_table;
740 }
741 if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_MCLK,
742 &arb->mclk_min,
743 &arb->mclk_max) < 0) {
744 nvgpu_err(g, "failed to fetch MCLK range");
745 goto exit_vf_table;
746 }
747
748 table->gpc2clk_num_points = MAX_F_POINTS;
749 table->mclk_num_points = MAX_F_POINTS;
750
751 if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_GPC2CLK,
752 &table->gpc2clk_num_points, arb->gpc2clk_f_points)) {
753 nvgpu_err(g, "failed to fetch GPC2CLK frequency points");
754 goto exit_vf_table;
755 }
756
757 if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_MCLK,
758 &table->mclk_num_points, arb->mclk_f_points)) {
759 nvgpu_err(g, "failed to fetch MCLK frequency points");
760 goto exit_vf_table;
761 }
762 if (!table->mclk_num_points || !table->gpc2clk_num_points) {
763 nvgpu_err(g, "empty queries to f points mclk %d gpc2clk %d",
764 table->mclk_num_points, table->gpc2clk_num_points);
765 status = -EINVAL;
766 goto exit_vf_table;
767 }
768
769 memset(table->mclk_points, 0,
770 table->mclk_num_points*sizeof(struct nvgpu_clk_vf_point));
771 memset(table->gpc2clk_points, 0,
772 table->gpc2clk_num_points*sizeof(struct nvgpu_clk_vf_point));
773
774 p5_info = pstate_get_clk_set_info(g,
775 CTRL_PERF_PSTATE_P5, clkwhich_mclk);
776 if (!p5_info) {
777 nvgpu_err(g, "failed to get MCLK P5 info");
778 goto exit_vf_table;
779 }
780 p0_info = pstate_get_clk_set_info(g,
781 CTRL_PERF_PSTATE_P0, clkwhich_mclk);
782 if (!p0_info) {
783 nvgpu_err(g, "failed to get MCLK P0 info");
784 goto exit_vf_table;
785 }
786
787 for (i = 0, j = 0, num_points = 0, clk_cur = 0;
788 i < table->mclk_num_points; i++) {
789
790 if ((arb->mclk_f_points[i] >= arb->mclk_min) &&
791 (arb->mclk_f_points[i] <= arb->mclk_max) &&
792 (arb->mclk_f_points[i] != clk_cur)) {
793
794 table->mclk_points[j].mem_mhz = arb->mclk_f_points[i];
795 mclk_voltuv = mclk_voltuv_sram = 0;
796
797 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK,
798 &table->mclk_points[j].mem_mhz, &mclk_voltuv,
799 CTRL_VOLT_DOMAIN_LOGIC);
800 if (status < 0) {
801 nvgpu_err(g,
802 "failed to get MCLK LOGIC voltage");
803 goto exit_vf_table;
804 }
805 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK,
806 &table->mclk_points[j].mem_mhz,
807 &mclk_voltuv_sram,
808 CTRL_VOLT_DOMAIN_SRAM);
809 if (status < 0) {
810 nvgpu_err(g, "failed to get MCLK SRAM voltage");
811 goto exit_vf_table;
812 }
813
814 table->mclk_points[j].uvolt = mclk_voltuv;
815 table->mclk_points[j].uvolt_sram = mclk_voltuv_sram;
816 clk_cur = table->mclk_points[j].mem_mhz;
817
818 if ((clk_cur >= p5_info->min_mhz) &&
819 (clk_cur <= p5_info->max_mhz))
820 VF_POINT_SET_PSTATE_SUPPORTED(
821 &table->mclk_points[j],
822 CTRL_PERF_PSTATE_P5);
823 if ((clk_cur >= p0_info->min_mhz) &&
824 (clk_cur <= p0_info->max_mhz))
825 VF_POINT_SET_PSTATE_SUPPORTED(
826 &table->mclk_points[j],
827 CTRL_PERF_PSTATE_P0);
828
829 j++;
830 num_points++;
831
832 }
833 }
834 table->mclk_num_points = num_points;
835
836 p5_info = pstate_get_clk_set_info(g,
837 CTRL_PERF_PSTATE_P5, clkwhich_gpc2clk);
838 if (!p5_info) {
839 status = -EINVAL;
840 nvgpu_err(g, "failed to get GPC2CLK P5 info");
841 goto exit_vf_table;
842 }
843
844 p0_info = pstate_get_clk_set_info(g,
845 CTRL_PERF_PSTATE_P0, clkwhich_gpc2clk);
846 if (!p0_info) {
847 status = -EINVAL;
848 nvgpu_err(g, "failed to get GPC2CLK P0 info");
849 goto exit_vf_table;
850 }
851
852 /* GPC2CLK needs to be checked in two passes. The first determines the
853 * relationships between GPC2CLK, SYS2CLK and XBAR2CLK, while the
854 * second verifies that the clocks minimum is satisfied and sets
855 * the voltages
856 */
857 for (i = 0, j = 0, num_points = 0, clk_cur = 0;
858 i < table->gpc2clk_num_points; i++) {
859 struct set_fll_clk setfllclk;
860
861 if ((arb->gpc2clk_f_points[i] >= arb->gpc2clk_min) &&
862 (arb->gpc2clk_f_points[i] <= arb->gpc2clk_max) &&
863 (arb->gpc2clk_f_points[i] != clk_cur)) {
864
865 table->gpc2clk_points[j].gpc_mhz =
866 arb->gpc2clk_f_points[i];
867 setfllclk.gpc2clkmhz = arb->gpc2clk_f_points[i];
868 status = clk_get_fll_clks(g, &setfllclk);
869 if (status < 0) {
870 nvgpu_err(g,
871 "failed to get GPC2CLK slave clocks");
872 goto exit_vf_table;
873 }
874
875 table->gpc2clk_points[j].sys_mhz =
876 setfllclk.sys2clkmhz;
877 table->gpc2clk_points[j].xbar_mhz =
878 setfllclk.xbar2clkmhz;
879
880 clk_cur = table->gpc2clk_points[j].gpc_mhz;
881
882 if ((clk_cur >= p5_info->min_mhz) &&
883 (clk_cur <= p5_info->max_mhz))
884 VF_POINT_SET_PSTATE_SUPPORTED(
885 &table->gpc2clk_points[j],
886 CTRL_PERF_PSTATE_P5);
887 if ((clk_cur >= p0_info->min_mhz) &&
888 (clk_cur <= p0_info->max_mhz))
889 VF_POINT_SET_PSTATE_SUPPORTED(
890 &table->gpc2clk_points[j],
891 CTRL_PERF_PSTATE_P0);
892
893 j++;
894 num_points++;
895 }
896 }
897 table->gpc2clk_num_points = num_points;
898
899 /* Second pass */
900 for (i = 0, j = 0; i < table->gpc2clk_num_points; i++) {
901
902 u16 alt_gpc2clk = table->gpc2clk_points[i].gpc_mhz;
903
904 gpc2clk_voltuv = gpc2clk_voltuv_sram = 0;
905
906 /* Check sysclk */
907 p5_info = pstate_get_clk_set_info(g,
908 VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]),
909 clkwhich_sys2clk);
910 if (!p5_info) {
911 status = -EINVAL;
912 nvgpu_err(g, "failed to get SYS2CLK P5 info");
913 goto exit_vf_table;
914 }
915
916 /* sys2clk below clk min, need to find correct clock */
917 if (table->gpc2clk_points[i].sys_mhz < p5_info->min_mhz) {
918 for (j = i + 1; j < table->gpc2clk_num_points; j++) {
919
920 if (table->gpc2clk_points[j].sys_mhz >=
921 p5_info->min_mhz) {
922
923
924 table->gpc2clk_points[i].sys_mhz =
925 p5_info->min_mhz;
926
927 alt_gpc2clk = alt_gpc2clk <
928 table->gpc2clk_points[j].
929 gpc_mhz ?
930 table->gpc2clk_points[j].
931 gpc_mhz :
932 alt_gpc2clk;
933 break;
934 }
935 }
936 /* no VF exists that satisfies condition */
937 if (j == table->gpc2clk_num_points) {
938 nvgpu_err(g, "NO SYS2CLK VF point possible");
939 status = -EINVAL;
940 goto exit_vf_table;
941 }
942 }
943
944 /* Check xbarclk */
945 p5_info = pstate_get_clk_set_info(g,
946 VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]),
947 clkwhich_xbar2clk);
948 if (!p5_info) {
949 status = -EINVAL;
950 nvgpu_err(g, "failed to get SYS2CLK P5 info");
951 goto exit_vf_table;
952 }
953
954 /* xbar2clk below clk min, need to find correct clock */
955 if (table->gpc2clk_points[i].xbar_mhz < p5_info->min_mhz) {
956 for (j = i; j < table->gpc2clk_num_points; j++) {
957 if (table->gpc2clk_points[j].xbar_mhz >=
958 p5_info->min_mhz) {
959
960 table->gpc2clk_points[i].xbar_mhz =
961 p5_info->min_mhz;
962
963 alt_gpc2clk = alt_gpc2clk <
964 table->gpc2clk_points[j].
965 gpc_mhz ?
966 table->gpc2clk_points[j].
967 gpc_mhz :
968 alt_gpc2clk;
969 break;
970 }
971 }
972 /* no VF exists that satisfies condition */
973 if (j == table->gpc2clk_num_points) {
974 status = -EINVAL;
975 nvgpu_err(g, "NO XBAR2CLK VF point possible");
976
977 goto exit_vf_table;
978 }
979 }
980
981 /* Calculate voltages */
982 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK,
983 &alt_gpc2clk, &gpc2clk_voltuv,
984 CTRL_VOLT_DOMAIN_LOGIC);
985 if (status < 0) {
986 nvgpu_err(g, "failed to get GPC2CLK LOGIC voltage");
987 goto exit_vf_table;
988 }
989
990 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK,
991 &alt_gpc2clk,
992 &gpc2clk_voltuv_sram,
993 CTRL_VOLT_DOMAIN_SRAM);
994 if (status < 0) {
995 nvgpu_err(g, "failed to get GPC2CLK SRAM voltage");
996 goto exit_vf_table;
997 }
998
999 table->gpc2clk_points[i].uvolt = gpc2clk_voltuv;
1000 table->gpc2clk_points[i].uvolt_sram = gpc2clk_voltuv_sram;
1001 }
1002
1003 /* make table visible when all data has resolved in the tables */
1004 nvgpu_smp_wmb();
1005 xchg(&arb->current_vf_table, table);
1006
1007exit_vf_table:
1008
1009 if (status < 0)
1010 nvgpu_clk_arb_set_global_alarm(g,
1011 EVENT(ALARM_VF_TABLE_UPDATE_FAILED));
1012 if (arb->update_work_queue)
1013 queue_work(arb->update_work_queue, &arb->update_fn_work);
1014
1015 return status;
1016}
1017
1018void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g)
1019{
1020 struct nvgpu_clk_arb *arb = g->clk_arb;
1021
1022 if (arb->vf_table_work_queue)
1023 queue_work(arb->vf_table_work_queue, &arb->vf_table_fn_work);
1024}
1025
1026static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work)
1027{
1028 struct nvgpu_clk_arb *arb =
1029 container_of(work, struct nvgpu_clk_arb, vf_table_fn_work);
1030 struct gk20a *g = arb->g;
1031 u32 err;
1032
1033 /* get latest vf curve from pmu */
1034 err = clk_vf_point_cache(g);
1035 if (err) {
1036 nvgpu_err(g, "failed to cache VF table");
1037 nvgpu_clk_arb_set_global_alarm(g,
1038 EVENT(ALARM_VF_TABLE_UPDATE_FAILED));
1039 if (arb->update_work_queue)
1040 queue_work(arb->update_work_queue,
1041 &arb->update_fn_work);
1042
1043 return;
1044 }
1045 nvgpu_clk_arb_update_vf_table(arb);
1046}
1047
1048static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work)
1049{
1050 struct nvgpu_clk_arb *arb =
1051 container_of(work, struct nvgpu_clk_arb, update_fn_work);
1052 struct nvgpu_clk_session *session;
1053 struct nvgpu_clk_dev *dev;
1054 struct nvgpu_clk_dev *tmp;
1055 struct nvgpu_clk_arb_target *target, *actual;
1056 struct gk20a *g = arb->g;
1057 struct llist_node *head;
1058
1059 u32 pstate = VF_POINT_INVALID_PSTATE;
1060 u32 voltuv, voltuv_sram;
1061 bool mclk_set, gpc2clk_set;
1062 u32 nuvmin, nuvmin_sram;
1063
1064 u32 alarms_notified = 0;
1065 u32 current_alarm;
1066 int status = 0;
1067
1068 /* Temporary variables for checking target frequency */
1069 u16 gpc2clk_target, sys2clk_target, xbar2clk_target, mclk_target;
1070 u16 gpc2clk_session_target, mclk_session_target;
1071
1072#ifdef CONFIG_DEBUG_FS
1073 u64 t0, t1;
1074 struct nvgpu_clk_arb_debug *debug;
1075
1076#endif
1077
1078 gk20a_dbg_fn("");
1079
1080 /* bail out if gpu is down */
1081 if (nvgpu_atomic64_read(&arb->alarm_mask) & EVENT(ALARM_GPU_LOST))
1082 goto exit_arb;
1083
1084#ifdef CONFIG_DEBUG_FS
1085 g->ops.bus.read_ptimer(g, &t0);
1086#endif
1087
1088 /* Only one arbiter should be running */
1089 gpc2clk_target = 0;
1090 mclk_target = 0;
1091
1092 rcu_read_lock();
1093 list_for_each_entry_rcu(session, &arb->sessions, link) {
1094 if (!session->zombie) {
1095 mclk_set = false;
1096 gpc2clk_set = false;
1097 target = NV_ACCESS_ONCE(session->target) ==
1098 &session->target_pool[0] ?
1099 &session->target_pool[1] :
1100 &session->target_pool[0];
1101 /* Do not reorder pointer */
1102 nvgpu_smp_rmb();
1103 head = llist_del_all(&session->targets);
1104 if (head) {
1105
1106 /* Copy over state */
1107 target->mclk = session->target->mclk;
1108 target->gpc2clk = session->target->gpc2clk;
1109 /* Query the latest committed request */
1110 llist_for_each_entry_safe(dev, tmp, head,
1111 node) {
1112 if (!mclk_set && dev->mclk_target_mhz) {
1113 target->mclk =
1114 dev->mclk_target_mhz;
1115 mclk_set = true;
1116 }
1117 if (!gpc2clk_set &&
1118 dev->gpc2clk_target_mhz) {
1119 target->gpc2clk =
1120 dev->gpc2clk_target_mhz;
1121 gpc2clk_set = true;
1122 }
1123 nvgpu_ref_get(&dev->refcount);
1124 llist_add(&dev->node, &arb->requests);
1125 }
1126 /* Ensure target is updated before ptr sawp */
1127 nvgpu_smp_wmb();
1128 xchg(&session->target, target);
1129 }
1130
1131 mclk_target = mclk_target > session->target->mclk ?
1132 mclk_target : session->target->mclk;
1133
1134 gpc2clk_target =
1135 gpc2clk_target > session->target->gpc2clk ?
1136 gpc2clk_target : session->target->gpc2clk;
1137 }
1138 }
1139 rcu_read_unlock();
1140
1141 gpc2clk_target = (gpc2clk_target > 0) ? gpc2clk_target :
1142 arb->gpc2clk_default_mhz;
1143
1144 if (gpc2clk_target < arb->gpc2clk_min)
1145 gpc2clk_target = arb->gpc2clk_min;
1146
1147 if (gpc2clk_target > arb->gpc2clk_max)
1148 gpc2clk_target = arb->gpc2clk_max;
1149
1150 mclk_target = (mclk_target > 0) ? mclk_target :
1151 arb->mclk_default_mhz;
1152
1153 if (mclk_target < arb->mclk_min)
1154 mclk_target = arb->mclk_min;
1155
1156 if (mclk_target > arb->mclk_max)
1157 mclk_target = arb->mclk_max;
1158
1159 sys2clk_target = 0;
1160 xbar2clk_target = 0;
1161
1162 gpc2clk_session_target = gpc2clk_target;
1163 mclk_session_target = mclk_target;
1164
1165 /* Query the table for the closest vf point to program */
1166 pstate = nvgpu_clk_arb_find_vf_point(arb, &gpc2clk_target,
1167 &sys2clk_target, &xbar2clk_target, &mclk_target, &voltuv,
1168 &voltuv_sram, &nuvmin, &nuvmin_sram);
1169
1170 if (pstate == VF_POINT_INVALID_PSTATE) {
1171 arb->status = -EINVAL;
1172 /* make status visible */
1173 nvgpu_smp_mb();
1174 goto exit_arb;
1175 }
1176
1177 if ((gpc2clk_target < gpc2clk_session_target) ||
1178 (mclk_target < mclk_session_target))
1179 nvgpu_clk_arb_set_global_alarm(g,
1180 EVENT(ALARM_TARGET_VF_NOT_POSSIBLE));
1181
1182 if ((arb->actual->gpc2clk == gpc2clk_target) &&
1183 (arb->actual->mclk == mclk_target) &&
1184 (arb->voltuv_actual == voltuv)) {
1185 goto exit_arb;
1186 }
1187
1188 /* Program clocks */
1189 /* A change in both mclk of gpc2clk may require a change in voltage */
1190
1191 nvgpu_mutex_acquire(&arb->pstate_lock);
1192 status = nvgpu_lpwr_disable_pg(g, false);
1193
1194 status = clk_pmu_freq_controller_load(g, false,
1195 CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL);
1196 if (status < 0) {
1197 arb->status = status;
1198 nvgpu_mutex_release(&arb->pstate_lock);
1199
1200 /* make status visible */
1201 nvgpu_smp_mb();
1202 goto exit_arb;
1203 }
1204 status = volt_set_noiseaware_vmin(g, nuvmin, nuvmin_sram);
1205 if (status < 0) {
1206 arb->status = status;
1207 nvgpu_mutex_release(&arb->pstate_lock);
1208
1209 /* make status visible */
1210 nvgpu_smp_mb();
1211 goto exit_arb;
1212 }
1213
1214 status = nvgpu_clk_arb_change_vf_point(g, gpc2clk_target,
1215 sys2clk_target, xbar2clk_target, mclk_target, voltuv,
1216 voltuv_sram);
1217 if (status < 0) {
1218 arb->status = status;
1219 nvgpu_mutex_release(&arb->pstate_lock);
1220
1221 /* make status visible */
1222 nvgpu_smp_mb();
1223 goto exit_arb;
1224 }
1225
1226 status = clk_pmu_freq_controller_load(g, true,
1227 CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL);
1228 if (status < 0) {
1229 arb->status = status;
1230 nvgpu_mutex_release(&arb->pstate_lock);
1231
1232 /* make status visible */
1233 nvgpu_smp_mb();
1234 goto exit_arb;
1235 }
1236
1237 status = nvgpu_lwpr_mclk_change(g, pstate);
1238 if (status < 0) {
1239 arb->status = status;
1240 nvgpu_mutex_release(&arb->pstate_lock);
1241
1242 /* make status visible */
1243 nvgpu_smp_mb();
1244 goto exit_arb;
1245 }
1246
1247 actual = NV_ACCESS_ONCE(arb->actual) == &arb->actual_pool[0] ?
1248 &arb->actual_pool[1] : &arb->actual_pool[0];
1249
1250 /* do not reorder this pointer */
1251 nvgpu_smp_rmb();
1252 actual->gpc2clk = gpc2clk_target;
1253 actual->mclk = mclk_target;
1254 arb->voltuv_actual = voltuv;
1255 actual->pstate = pstate;
1256 arb->status = status;
1257
1258 /* Make changes visible to other threads */
1259 nvgpu_smp_wmb();
1260 xchg(&arb->actual, actual);
1261
1262 status = nvgpu_lpwr_enable_pg(g, false);
1263 if (status < 0) {
1264 arb->status = status;
1265 nvgpu_mutex_release(&arb->pstate_lock);
1266
1267 /* make status visible */
1268 nvgpu_smp_mb();
1269 goto exit_arb;
1270 }
1271
1272 /* status must be visible before atomic inc */
1273 nvgpu_smp_wmb();
1274 nvgpu_atomic_inc(&arb->req_nr);
1275
1276 /* Unlock pstate change for PG */
1277 nvgpu_mutex_release(&arb->pstate_lock);
1278
1279 /* VF Update complete */
1280 nvgpu_clk_arb_set_global_alarm(g, EVENT(VF_UPDATE));
1281
1282 nvgpu_cond_signal_interruptible(&arb->request_wq);
1283
1284#ifdef CONFIG_DEBUG_FS
1285 g->ops.bus.read_ptimer(g, &t1);
1286
1287 debug = arb->debug == &arb->debug_pool[0] ?
1288 &arb->debug_pool[1] : &arb->debug_pool[0];
1289
1290 memcpy(debug, arb->debug, sizeof(arb->debug_pool[0]));
1291 debug->switch_num++;
1292
1293 if (debug->switch_num == 1) {
1294 debug->switch_max = debug->switch_min =
1295 debug->switch_avg = (t1-t0)/1000;
1296 debug->switch_std = 0;
1297 } else {
1298 s64 prev_avg;
1299 s64 curr = (t1-t0)/1000;
1300
1301 debug->switch_max = curr > debug->switch_max ?
1302 curr : debug->switch_max;
1303 debug->switch_min = debug->switch_min ?
1304 (curr < debug->switch_min ?
1305 curr : debug->switch_min) : curr;
1306 prev_avg = debug->switch_avg;
1307 debug->switch_avg = (curr +
1308 (debug->switch_avg * (debug->switch_num-1))) /
1309 debug->switch_num;
1310 debug->switch_std +=
1311 (curr - debug->switch_avg) * (curr - prev_avg);
1312 }
1313 /* commit changes before exchanging debug pointer */
1314 nvgpu_smp_wmb();
1315 xchg(&arb->debug, debug);
1316#endif
1317
1318exit_arb:
1319 if (status < 0) {
1320 nvgpu_err(g, "Error in arbiter update");
1321 nvgpu_clk_arb_set_global_alarm(g,
1322 EVENT(ALARM_CLOCK_ARBITER_FAILED));
1323 }
1324
1325 current_alarm = (u32) nvgpu_atomic64_read(&arb->alarm_mask);
1326 /* notify completion for all requests */
1327 head = llist_del_all(&arb->requests);
1328 llist_for_each_entry_safe(dev, tmp, head, node) {
1329 nvgpu_atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM);
1330 nvgpu_cond_signal_interruptible(&dev->readout_wq);
1331 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1332 }
1333
1334 nvgpu_atomic_set(&arb->notification_queue.head,
1335 nvgpu_atomic_read(&arb->notification_queue.tail));
1336 /* notify event for all users */
1337 rcu_read_lock();
1338 list_for_each_entry_rcu(dev, &arb->users, link) {
1339 alarms_notified |=
1340 nvgpu_clk_arb_notify(dev, arb->actual, current_alarm);
1341 }
1342 rcu_read_unlock();
1343
1344 /* clear alarms */
1345 nvgpu_clk_arb_clear_global_alarm(g, alarms_notified &
1346 ~EVENT(ALARM_GPU_LOST));
1347}
1348
1349static void nvgpu_clk_arb_queue_notification(struct gk20a *g,
1350 struct nvgpu_clk_notification_queue *queue,
1351 u32 alarm_mask) {
1352
1353 u32 queue_index;
1354 u64 timestamp;
1355
1356 queue_index = (nvgpu_atomic_inc_return(&queue->tail)) % queue->size;
1357 /* get current timestamp */
1358 timestamp = (u64) sched_clock();
1359
1360 queue->notifications[queue_index].timestamp = timestamp;
1361 queue->notifications[queue_index].notification = alarm_mask;
1362
1363}
1364
1365static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev,
1366 struct nvgpu_clk_arb_target *target,
1367 u32 alarm) {
1368
1369 struct nvgpu_clk_session *session = dev->session;
1370 struct nvgpu_clk_arb *arb = session->g->clk_arb;
1371 struct nvgpu_clk_notification *notification;
1372
1373 u32 queue_alarm_mask = 0;
1374 u32 enabled_mask = 0;
1375 u32 new_alarms_reported = 0;
1376 u32 poll_mask = 0;
1377 u32 tail, head;
1378 u32 queue_index;
1379 size_t size;
1380 int index;
1381
1382 enabled_mask = nvgpu_atomic_read(&dev->enabled_mask);
1383 size = arb->notification_queue.size;
1384
1385 /* queue global arbiter notifications in buffer */
1386 do {
1387 tail = nvgpu_atomic_read(&arb->notification_queue.tail);
1388 /* copy items to the queue */
1389 queue_index = nvgpu_atomic_read(&dev->queue.tail);
1390 head = dev->arb_queue_head;
1391 head = (tail - head) < arb->notification_queue.size ?
1392 head : tail - arb->notification_queue.size;
1393
1394 for (index = head; _WRAPGTEQ(tail, index); index++) {
1395 u32 alarm_detected;
1396
1397 notification = &arb->notification_queue.
1398 notifications[(index+1) % size];
1399 alarm_detected =
1400 NV_ACCESS_ONCE(notification->notification);
1401
1402 if (!(enabled_mask & alarm_detected))
1403 continue;
1404
1405 queue_index++;
1406 dev->queue.notifications[
1407 queue_index % dev->queue.size].timestamp =
1408 NV_ACCESS_ONCE(notification->timestamp);
1409
1410 dev->queue.notifications[
1411 queue_index % dev->queue.size].notification =
1412 alarm_detected;
1413
1414 queue_alarm_mask |= alarm_detected;
1415 }
1416 } while (unlikely(nvgpu_atomic_read(&arb->notification_queue.tail) !=
1417 (int)tail));
1418
1419 nvgpu_atomic_set(&dev->queue.tail, queue_index);
1420 /* update the last notification we processed from global queue */
1421
1422 dev->arb_queue_head = tail;
1423
1424 /* Check if current session targets are met */
1425 if (enabled_mask & EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)) {
1426 if ((target->gpc2clk < session->target->gpc2clk)
1427 || (target->mclk < session->target->mclk)) {
1428
1429 poll_mask |= (POLLIN | POLLPRI);
1430 nvgpu_clk_arb_queue_notification(arb->g, &dev->queue,
1431 EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE));
1432 }
1433 }
1434
1435 /* Check if there is a new VF update */
1436 if (queue_alarm_mask & EVENT(VF_UPDATE))
1437 poll_mask |= (POLLIN | POLLRDNORM);
1438
1439 /* Notify sticky alarms that were not reported on previous run*/
1440 new_alarms_reported = (queue_alarm_mask |
1441 (alarm & ~dev->alarms_reported & queue_alarm_mask));
1442
1443 if (new_alarms_reported & ~LOCAL_ALARM_MASK) {
1444 /* check that we are not re-reporting */
1445 if (new_alarms_reported & EVENT(ALARM_GPU_LOST))
1446 poll_mask |= POLLHUP;
1447
1448 poll_mask |= (POLLIN | POLLPRI);
1449 /* On next run do not report global alarms that were already
1450 * reported, but report SHUTDOWN always
1451 */
1452 dev->alarms_reported = new_alarms_reported & ~LOCAL_ALARM_MASK &
1453 ~EVENT(ALARM_GPU_LOST);
1454 }
1455
1456 if (poll_mask) {
1457 nvgpu_atomic_set(&dev->poll_mask, poll_mask);
1458 nvgpu_cond_broadcast_interruptible(&dev->readout_wq);
1459 }
1460
1461 return new_alarms_reported;
1462}
1463
1464static int nvgpu_clk_arb_set_event_filter(struct nvgpu_clk_dev *dev,
1465 struct nvgpu_gpu_set_event_filter_args *args)
1466{
1467 u32 mask;
1468
1469 gk20a_dbg(gpu_dbg_fn, "");
1470
1471 if (args->flags)
1472 return -EINVAL;
1473
1474 if (args->size != 1)
1475 return -EINVAL;
1476
1477 if (copy_from_user(&mask, (void __user *) args->buffer,
1478 args->size * sizeof(u32)))
1479 return -EFAULT;
1480
1481 /* update alarm mask */
1482 nvgpu_atomic_set(&dev->enabled_mask, mask);
1483
1484 return 0;
1485}
1486
1487static long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd,
1488 unsigned long arg)
1489{
1490 struct nvgpu_clk_dev *dev = filp->private_data;
1491 struct gk20a *g = dev->session->g;
1492 u8 buf[NVGPU_EVENT_IOCTL_MAX_ARG_SIZE];
1493 int err = 0;
1494
1495 gk20a_dbg(gpu_dbg_fn, "nr=%d", _IOC_NR(cmd));
1496
1497 if ((_IOC_TYPE(cmd) != NVGPU_EVENT_IOCTL_MAGIC) || (_IOC_NR(cmd) == 0)
1498 || (_IOC_NR(cmd) > NVGPU_EVENT_IOCTL_LAST))
1499 return -EINVAL;
1500
1501 BUG_ON(_IOC_SIZE(cmd) > NVGPU_EVENT_IOCTL_MAX_ARG_SIZE);
1502
1503 memset(buf, 0, sizeof(buf));
1504 if (_IOC_DIR(cmd) & _IOC_WRITE) {
1505 if (copy_from_user(buf, (void __user *) arg, _IOC_SIZE(cmd)))
1506 return -EFAULT;
1507 }
1508
1509 switch (cmd) {
1510 case NVGPU_EVENT_IOCTL_SET_FILTER:
1511 err = nvgpu_clk_arb_set_event_filter(dev,
1512 (struct nvgpu_gpu_set_event_filter_args *)buf);
1513 break;
1514 default:
1515 nvgpu_warn(g, "unrecognized event ioctl cmd: 0x%x", cmd);
1516 err = -ENOTTY;
1517 }
1518
1519 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1520 err = copy_to_user((void __user *) arg, buf, _IOC_SIZE(cmd));
1521
1522 return err;
1523}
1524
1525int nvgpu_clk_arb_commit_request_fd(struct gk20a *g,
1526 struct nvgpu_clk_session *session, int request_fd)
1527{
1528 struct nvgpu_clk_arb *arb = g->clk_arb;
1529 struct nvgpu_clk_dev *dev;
1530 struct fd fd;
1531 int err = 0;
1532
1533 gk20a_dbg_fn("");
1534
1535 fd = fdget(request_fd);
1536 if (!fd.file)
1537 return -EINVAL;
1538
1539 if (fd.file->f_op != &completion_dev_ops) {
1540 err = -EINVAL;
1541 goto fdput_fd;
1542 }
1543
1544 dev = (struct nvgpu_clk_dev *) fd.file->private_data;
1545
1546 if (!dev || dev->session != session) {
1547 err = -EINVAL;
1548 goto fdput_fd;
1549 }
1550 nvgpu_ref_get(&dev->refcount);
1551 llist_add(&dev->node, &session->targets);
1552 if (arb->update_work_queue)
1553 queue_work(arb->update_work_queue, &arb->update_fn_work);
1554
1555fdput_fd:
1556 fdput(fd);
1557 return err;
1558}
1559
1560static inline u32 __pending_event(struct nvgpu_clk_dev *dev,
1561 struct nvgpu_gpu_event_info *info) {
1562
1563 u32 tail, head;
1564 u32 events = 0;
1565 struct nvgpu_clk_notification *p_notif;
1566
1567 tail = nvgpu_atomic_read(&dev->queue.tail);
1568 head = nvgpu_atomic_read(&dev->queue.head);
1569
1570 head = (tail - head) < dev->queue.size ? head : tail - dev->queue.size;
1571
1572 if (_WRAPGTEQ(tail, head) && info) {
1573 head++;
1574 p_notif = &dev->queue.notifications[head % dev->queue.size];
1575 events |= p_notif->notification;
1576 info->event_id = ffs(events) - 1;
1577 info->timestamp = p_notif->timestamp;
1578 nvgpu_atomic_set(&dev->queue.head, head);
1579 }
1580
1581 return events;
1582}
1583
1584static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf,
1585 size_t size, loff_t *off)
1586{
1587 struct nvgpu_clk_dev *dev = filp->private_data;
1588 struct nvgpu_gpu_event_info info;
1589 ssize_t err;
1590
1591 gk20a_dbg_fn("filp=%p, buf=%p, size=%zu", filp, buf, size);
1592
1593 if ((size - *off) < sizeof(info))
1594 return 0;
1595
1596 memset(&info, 0, sizeof(info));
1597 /* Get the oldest event from the queue */
1598 while (!__pending_event(dev, &info)) {
1599 if (filp->f_flags & O_NONBLOCK)
1600 return -EAGAIN;
1601 err = NVGPU_COND_WAIT_INTERRUPTIBLE(&dev->readout_wq,
1602 __pending_event(dev, &info), 0);
1603 if (err)
1604 return err;
1605 if (info.timestamp)
1606 break;
1607 }
1608
1609 if (copy_to_user(buf + *off, &info, sizeof(info)))
1610 return -EFAULT;
1611
1612 return sizeof(info);
1613}
1614
1615static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait)
1616{
1617 struct nvgpu_clk_dev *dev = filp->private_data;
1618
1619 gk20a_dbg_fn("");
1620
1621 poll_wait(filp, &dev->readout_wq.wq, wait);
1622 return nvgpu_atomic_xchg(&dev->poll_mask, 0);
1623}
1624
1625static int nvgpu_clk_arb_release_completion_dev(struct inode *inode,
1626 struct file *filp)
1627{
1628 struct nvgpu_clk_dev *dev = filp->private_data;
1629 struct nvgpu_clk_session *session = dev->session;
1630
1631
1632 gk20a_dbg_fn("");
1633
1634 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
1635 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1636 return 0;
1637}
1638
1639static int nvgpu_clk_arb_release_event_dev(struct inode *inode,
1640 struct file *filp)
1641{
1642 struct nvgpu_clk_dev *dev = filp->private_data;
1643 struct nvgpu_clk_session *session = dev->session;
1644 struct nvgpu_clk_arb *arb;
1645
1646 arb = session->g->clk_arb;
1647
1648 gk20a_dbg_fn("");
1649
1650 if (arb) {
1651 nvgpu_spinlock_acquire(&arb->users_lock);
1652 list_del_rcu(&dev->link);
1653 nvgpu_spinlock_release(&arb->users_lock);
1654 nvgpu_clk_notification_queue_free(arb->g, &dev->queue);
1655 }
1656
1657 synchronize_rcu();
1658 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
1659 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1660
1661 return 0;
1662}
1663
1664int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session,
1665 int request_fd, u32 api_domain, u16 target_mhz)
1666{
1667 struct nvgpu_clk_dev *dev;
1668 struct fd fd;
1669 int err = 0;
1670
1671 gk20a_dbg_fn("domain=0x%08x target_mhz=%u", api_domain, target_mhz);
1672
1673 fd = fdget(request_fd);
1674 if (!fd.file)
1675 return -EINVAL;
1676
1677 if (fd.file->f_op != &completion_dev_ops) {
1678 err = -EINVAL;
1679 goto fdput_fd;
1680 }
1681
1682 dev = fd.file->private_data;
1683 if (!dev || dev->session != session) {
1684 err = -EINVAL;
1685 goto fdput_fd;
1686 }
1687
1688 switch (api_domain) {
1689 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1690 dev->mclk_target_mhz = target_mhz;
1691 break;
1692
1693 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1694 dev->gpc2clk_target_mhz = target_mhz * 2ULL;
1695 break;
1696
1697 default:
1698 err = -EINVAL;
1699 }
1700
1701fdput_fd:
1702 fdput(fd);
1703 return err;
1704}
1705
1706int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session,
1707 u32 api_domain, u16 *freq_mhz)
1708{
1709 int err = 0;
1710 struct nvgpu_clk_arb_target *target;
1711
1712 do {
1713 target = NV_ACCESS_ONCE(session->target);
1714 /* no reordering of this pointer */
1715 nvgpu_smp_rmb();
1716
1717 switch (api_domain) {
1718 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1719 *freq_mhz = target->mclk;
1720 break;
1721
1722 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1723 *freq_mhz = target->gpc2clk / 2ULL;
1724 break;
1725
1726 default:
1727 *freq_mhz = 0;
1728 err = -EINVAL;
1729 }
1730 } while (target != NV_ACCESS_ONCE(session->target));
1731 return err;
1732}
1733
1734int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g,
1735 u32 api_domain, u16 *freq_mhz)
1736{
1737 struct nvgpu_clk_arb *arb = g->clk_arb;
1738 int err = 0;
1739 struct nvgpu_clk_arb_target *actual;
1740
1741 do {
1742 actual = NV_ACCESS_ONCE(arb->actual);
1743 /* no reordering of this pointer */
1744 nvgpu_smp_rmb();
1745
1746 switch (api_domain) {
1747 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1748 *freq_mhz = actual->mclk;
1749 break;
1750
1751 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1752 *freq_mhz = actual->gpc2clk / 2ULL;
1753 break;
1754
1755 default:
1756 *freq_mhz = 0;
1757 err = -EINVAL;
1758 }
1759 } while (actual != NV_ACCESS_ONCE(arb->actual));
1760 return err;
1761}
1762
1763int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g,
1764 u32 api_domain, u16 *freq_mhz)
1765{
1766 switch (api_domain) {
1767 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1768 *freq_mhz = g->ops.clk.measure_freq(g, CTRL_CLK_DOMAIN_MCLK) /
1769 1000000ULL;
1770 return 0;
1771
1772 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1773 *freq_mhz = g->ops.clk.measure_freq(g,
1774 CTRL_CLK_DOMAIN_GPC2CLK) / 2000000ULL;
1775 return 0;
1776
1777 default:
1778 return -EINVAL;
1779 }
1780}
1781
1782int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
1783 u16 *min_mhz, u16 *max_mhz)
1784{
1785 int ret;
1786
1787 switch (api_domain) {
1788 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1789 ret = g->ops.clk_arb.get_arbiter_clk_range(g,
1790 CTRL_CLK_DOMAIN_MCLK, min_mhz, max_mhz);
1791 return ret;
1792
1793 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1794 ret = g->ops.clk_arb.get_arbiter_clk_range(g,
1795 CTRL_CLK_DOMAIN_GPC2CLK, min_mhz, max_mhz);
1796 if (!ret) {
1797 *min_mhz /= 2;
1798 *max_mhz /= 2;
1799 }
1800 return ret;
1801
1802 default:
1803 return -EINVAL;
1804 }
1805}
1806
1807u32 nvgpu_clk_arb_get_arbiter_clk_domains(struct gk20a *g)
1808{
1809 u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g);
1810 u32 api_domains = 0;
1811
1812 if (clk_domains & CTRL_CLK_DOMAIN_GPC2CLK)
1813 api_domains |= BIT(NVGPU_GPU_CLK_DOMAIN_GPCCLK);
1814
1815 if (clk_domains & CTRL_CLK_DOMAIN_MCLK)
1816 api_domains |= BIT(NVGPU_GPU_CLK_DOMAIN_MCLK);
1817
1818 return api_domains;
1819}
1820
1821bool nvgpu_clk_arb_is_valid_domain(struct gk20a *g, u32 api_domain)
1822{
1823 u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g);
1824
1825 switch (api_domain) {
1826 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1827 return ((clk_domains & CTRL_CLK_DOMAIN_MCLK) != 0);
1828
1829 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1830 return ((clk_domains & CTRL_CLK_DOMAIN_GPC2CLK) != 0);
1831
1832 default:
1833 return false;
1834 }
1835}
1836
1837int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g,
1838 u32 api_domain, u32 *max_points, u16 *fpoints)
1839{
1840 int err;
1841 u32 i;
1842
1843 switch (api_domain) {
1844 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1845 err = clk_domain_get_f_points(g, CTRL_CLK_DOMAIN_GPC2CLK,
1846 max_points, fpoints);
1847 if (err || !fpoints)
1848 return err;
1849 for (i = 0; i < *max_points; i++)
1850 fpoints[i] /= 2;
1851 return 0;
1852 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1853 return clk_domain_get_f_points(g, CTRL_CLK_DOMAIN_MCLK,
1854 max_points, fpoints);
1855 default:
1856 return -EINVAL;
1857 }
1858}
1859
1860static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb,
1861 u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk,
1862 u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram)
1863{
1864 u16 gpc2clk_target, mclk_target;
1865 u32 gpc2clk_voltuv, gpc2clk_voltuv_sram;
1866 u32 mclk_voltuv, mclk_voltuv_sram;
1867 u32 pstate = VF_POINT_INVALID_PSTATE;
1868 struct nvgpu_clk_vf_table *table;
1869 u32 index, index_mclk;
1870 struct nvgpu_clk_vf_point *mclk_vf = NULL;
1871
1872 do {
1873 gpc2clk_target = *gpc2clk;
1874 mclk_target = *mclk;
1875 gpc2clk_voltuv = 0;
1876 gpc2clk_voltuv_sram = 0;
1877 mclk_voltuv = 0;
1878 mclk_voltuv_sram = 0;
1879
1880 table = NV_ACCESS_ONCE(arb->current_vf_table);
1881 /* pointer to table can be updated by callback */
1882 nvgpu_smp_rmb();
1883
1884 if (!table)
1885 continue;
1886 if ((!table->gpc2clk_num_points) || (!table->mclk_num_points)) {
1887 nvgpu_err(arb->g, "found empty table");
1888 goto find_exit;
1889 }
1890 /* First we check MCLK to find out which PSTATE we are
1891 * are requesting, and from there try to find the minimum
1892 * GPC2CLK on the same PSTATE that satisfies the request.
1893 * If no GPC2CLK can be found, then we need to up the PSTATE
1894 */
1895
1896recalculate_vf_point:
1897 for (index = 0; index < table->mclk_num_points; index++) {
1898 if (table->mclk_points[index].mem_mhz >= mclk_target) {
1899 mclk_vf = &table->mclk_points[index];
1900 break;
1901 }
1902 }
1903 if (index == table->mclk_num_points) {
1904 mclk_vf = &table->mclk_points[index-1];
1905 index = table->mclk_num_points - 1;
1906 }
1907 index_mclk = index;
1908
1909 /* round up the freq requests */
1910 for (index = 0; index < table->gpc2clk_num_points; index++) {
1911 pstate = VF_POINT_COMMON_PSTATE(
1912 &table->gpc2clk_points[index], mclk_vf);
1913
1914 if ((table->gpc2clk_points[index].gpc_mhz >=
1915 gpc2clk_target) &&
1916 (pstate != VF_POINT_INVALID_PSTATE)) {
1917 gpc2clk_target =
1918 table->gpc2clk_points[index].gpc_mhz;
1919 *sys2clk =
1920 table->gpc2clk_points[index].sys_mhz;
1921 *xbar2clk =
1922 table->gpc2clk_points[index].xbar_mhz;
1923
1924 gpc2clk_voltuv =
1925 table->gpc2clk_points[index].uvolt;
1926 gpc2clk_voltuv_sram =
1927 table->gpc2clk_points[index].uvolt_sram;
1928 break;
1929 }
1930 }
1931
1932 if (index == table->gpc2clk_num_points) {
1933 pstate = VF_POINT_COMMON_PSTATE(
1934 &table->gpc2clk_points[index-1], mclk_vf);
1935 if (pstate != VF_POINT_INVALID_PSTATE) {
1936 gpc2clk_target =
1937 table->gpc2clk_points[index-1].gpc_mhz;
1938 *sys2clk =
1939 table->gpc2clk_points[index-1].sys_mhz;
1940 *xbar2clk =
1941 table->gpc2clk_points[index-1].xbar_mhz;
1942
1943 gpc2clk_voltuv =
1944 table->gpc2clk_points[index-1].uvolt;
1945 gpc2clk_voltuv_sram =
1946 table->gpc2clk_points[index-1].
1947 uvolt_sram;
1948 } else if (index_mclk >= table->mclk_num_points - 1) {
1949 /* There is no available combination of MCLK
1950 * and GPC2CLK, we need to fail this
1951 */
1952 gpc2clk_target = 0;
1953 mclk_target = 0;
1954 pstate = VF_POINT_INVALID_PSTATE;
1955 goto find_exit;
1956 } else {
1957 /* recalculate with higher PSTATE */
1958 gpc2clk_target = *gpc2clk;
1959 mclk_target = table->mclk_points[index_mclk+1].
1960 mem_mhz;
1961 goto recalculate_vf_point;
1962 }
1963 }
1964
1965 mclk_target = mclk_vf->mem_mhz;
1966 mclk_voltuv = mclk_vf->uvolt;
1967 mclk_voltuv_sram = mclk_vf->uvolt_sram;
1968
1969 } while (!table ||
1970 (NV_ACCESS_ONCE(arb->current_vf_table) != table));
1971
1972find_exit:
1973 *voltuv = gpc2clk_voltuv > mclk_voltuv ? gpc2clk_voltuv : mclk_voltuv;
1974 *voltuv_sram = gpc2clk_voltuv_sram > mclk_voltuv_sram ?
1975 gpc2clk_voltuv_sram : mclk_voltuv_sram;
1976 /* noise unaware vmin */
1977 *nuvmin = mclk_voltuv;
1978 *nuvmin_sram = mclk_voltuv_sram;
1979 *gpc2clk = gpc2clk_target < *gpc2clk ? gpc2clk_target : *gpc2clk;
1980 *mclk = mclk_target;
1981 return pstate;
1982}
1983
1984/* This function is inherently unsafe to call while arbiter is running
1985 * arbiter must be blocked before calling this function
1986 */
1987int nvgpu_clk_arb_get_current_pstate(struct gk20a *g)
1988{
1989 return NV_ACCESS_ONCE(g->clk_arb->actual->pstate);
1990}
1991
1992static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target,
1993 u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv,
1994 u32 voltuv_sram)
1995{
1996 struct set_fll_clk fllclk;
1997 struct nvgpu_clk_arb *arb = g->clk_arb;
1998 int status;
1999
2000 fllclk.gpc2clkmhz = gpc2clk_target;
2001 fllclk.sys2clkmhz = sys2clk_target;
2002 fllclk.xbar2clkmhz = xbar2clk_target;
2003
2004 fllclk.voltuv = voltuv;
2005
2006 /* if voltage ascends we do:
2007 * (1) FLL change
2008 * (2) Voltage change
2009 * (3) MCLK change
2010 * If it goes down
2011 * (1) MCLK change
2012 * (2) Voltage change
2013 * (3) FLL change
2014 */
2015
2016 /* descending */
2017 if (voltuv < arb->voltuv_actual) {
2018 status = g->ops.clk.mclk_change(g, mclk_target);
2019 if (status < 0)
2020 return status;
2021
2022 status = volt_set_voltage(g, voltuv, voltuv_sram);
2023 if (status < 0)
2024 return status;
2025
2026 status = clk_set_fll_clks(g, &fllclk);
2027 if (status < 0)
2028 return status;
2029 } else {
2030 status = clk_set_fll_clks(g, &fllclk);
2031 if (status < 0)
2032 return status;
2033
2034 status = volt_set_voltage(g, voltuv, voltuv_sram);
2035 if (status < 0)
2036 return status;
2037
2038 status = g->ops.clk.mclk_change(g, mclk_target);
2039 if (status < 0)
2040 return status;
2041 }
2042
2043 return 0;
2044}
2045
2046void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock)
2047{
2048 struct nvgpu_clk_arb *arb = g->clk_arb;
2049
2050 if (lock)
2051 nvgpu_mutex_acquire(&arb->pstate_lock);
2052 else
2053 nvgpu_mutex_release(&arb->pstate_lock);
2054}
2055
2056#ifdef CONFIG_DEBUG_FS
2057static int nvgpu_clk_arb_stats_show(struct seq_file *s, void *unused)
2058{
2059 struct gk20a *g = s->private;
2060 struct nvgpu_clk_arb *arb = g->clk_arb;
2061 struct nvgpu_clk_arb_debug *debug;
2062
2063 u64 num;
2064 s64 tmp, avg, std, max, min;
2065
2066 debug = NV_ACCESS_ONCE(arb->debug);
2067 /* Make copy of structure and ensure no reordering */
2068 nvgpu_smp_rmb();
2069 if (!debug)
2070 return -EINVAL;
2071
2072 std = debug->switch_std;
2073 avg = debug->switch_avg;
2074 max = debug->switch_max;
2075 min = debug->switch_min;
2076 num = debug->switch_num;
2077
2078 tmp = std;
2079 do_div(tmp, num);
2080 seq_printf(s, "Number of transitions: %lld\n",
2081 num);
2082 seq_printf(s, "max / min : %lld / %lld usec\n",
2083 max, min);
2084 seq_printf(s, "avg / std : %lld / %ld usec\n",
2085 avg, int_sqrt(tmp));
2086
2087 return 0;
2088}
2089
2090static int nvgpu_clk_arb_stats_open(struct inode *inode, struct file *file)
2091{
2092 return single_open(file, nvgpu_clk_arb_stats_show, inode->i_private);
2093}
2094
2095static const struct file_operations nvgpu_clk_arb_stats_fops = {
2096 .open = nvgpu_clk_arb_stats_open,
2097 .read = seq_read,
2098 .llseek = seq_lseek,
2099 .release = single_release,
2100};
2101
2102
2103static int nvgpu_clk_arb_debugfs_init(struct gk20a *g)
2104{
2105 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
2106 struct dentry *gpu_root = l->debugfs;
2107 struct dentry *d;
2108
2109 gk20a_dbg(gpu_dbg_info, "g=%p", g);
2110
2111 d = debugfs_create_file(
2112 "arb_stats",
2113 S_IRUGO,
2114 gpu_root,
2115 g,
2116 &nvgpu_clk_arb_stats_fops);
2117 if (!d)
2118 return -ENOMEM;
2119
2120 return 0;
2121}
2122#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-08-22 13:49:51 -0400