summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/clk
diff options
context:
space:
mode:
authorMahantesh Kumbar <mkumbar@nvidia.com>2017-10-20 03:04:00 -0400
committermobile promotions <svcmobile_promotions@nvidia.com>2017-10-25 20:29:36 -0400
commit0dcf0ede812aa55aa106a5e6c2f86216fcbfd5e0 (patch)
tree6414aa6c9966e98b7c58a9700a8e76abe1f93999 /drivers/gpu/nvgpu/clk
parentc79112f3b1e2a428603e06486bd3cea83942c14e (diff)
gpu: nvgpu: move clk_arb to linux specific
- Clock arbiter has lot of linux dependent code so moved clk_arb.c to common/linux folder & clk_arb.h to include/nvgpu/clk_arb.h, this move helps to unblock QNX. - QNX must implement functions present under clk_arb.h as needed. JIRA NVGPU-33 Change-Id: I38369fafda9c2cb9ba2175b3e530e40d0c746601 Signed-off-by: Mahantesh Kumbar <mkumbar@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/1582473 Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/clk')
-rw-r--r--drivers/gpu/nvgpu/clk/clk_arb.c2111
-rw-r--r--drivers/gpu/nvgpu/clk/clk_arb.h80
2 files changed, 0 insertions, 2191 deletions
diff --git a/drivers/gpu/nvgpu/clk/clk_arb.c b/drivers/gpu/nvgpu/clk/clk_arb.c
deleted file mode 100644
index 937d47db..00000000
--- a/drivers/gpu/nvgpu/clk/clk_arb.c
+++ /dev/null
@@ -1,2111 +0,0 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23
24#include <linux/cdev.h>
25#include <linux/file.h>
26#include <linux/anon_inodes.h>
27#include <linux/rculist.h>
28#include <linux/llist.h>
29#include <linux/uaccess.h>
30#include <linux/poll.h>
31#ifdef CONFIG_DEBUG_FS
32#include <linux/debugfs.h>
33#endif
34
35#include <nvgpu/bitops.h>
36#include <nvgpu/lock.h>
37#include <nvgpu/kmem.h>
38#include <nvgpu/atomic.h>
39#include <nvgpu/bug.h>
40#include <nvgpu/kref.h>
41#include <nvgpu/log.h>
42#include <nvgpu/barrier.h>
43#include <nvgpu/cond.h>
44
45#include "gk20a/gk20a.h"
46#include "clk/clk_arb.h"
47
48#ifdef CONFIG_DEBUG_FS
49#include "common/linux/os_linux.h"
50#endif
51
52#define MAX_F_POINTS 256
53#define DEFAULT_EVENT_NUMBER 32
54
55struct nvgpu_clk_dev;
56struct nvgpu_clk_arb_target;
57struct nvgpu_clk_notification_queue;
58
59#ifdef CONFIG_DEBUG_FS
60static int nvgpu_clk_arb_debugfs_init(struct gk20a *g);
61#endif
62
63static int nvgpu_clk_arb_release_event_dev(struct inode *inode,
64 struct file *filp);
65static int nvgpu_clk_arb_release_completion_dev(struct inode *inode,
66 struct file *filp);
67static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait);
68static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf,
69 size_t size, loff_t *off);
70
71static long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd,
72 unsigned long arg);
73
74static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work);
75static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work);
76static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb);
77static void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount);
78static void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount);
79static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target,
80 u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv,
81 u32 voltuv_sram);
82static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb,
83 u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk,
84 u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram);
85static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev,
86 struct nvgpu_clk_arb_target *target,
87 u32 alarm_mask);
88static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm);
89static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm);
90
91static void nvgpu_clk_arb_queue_notification(struct gk20a *g,
92 struct nvgpu_clk_notification_queue *queue,
93 u32 alarm_mask);
94static int nvgpu_clk_notification_queue_alloc(struct gk20a *g,
95 struct nvgpu_clk_notification_queue *queue,
96 size_t events_number);
97
98static void nvgpu_clk_notification_queue_free(struct gk20a *g,
99 struct nvgpu_clk_notification_queue *queue);
100
101#define VF_POINT_INVALID_PSTATE ~0U
102#define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b)))
103#define VF_POINT_GET_PSTATE(a) (((a)->pstates) ?\
104 __fls((a)->pstates) :\
105 VF_POINT_INVALID_PSTATE)
106#define VF_POINT_COMMON_PSTATE(a, b) (((a)->pstates & (b)->pstates) ?\
107 __fls((a)->pstates & (b)->pstates) :\
108 VF_POINT_INVALID_PSTATE)
109
110/* Local Alarms */
111#define EVENT(alarm) (0x1UL << NVGPU_GPU_EVENT_##alarm)
112
113#define LOCAL_ALARM_MASK (EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE) | \
114 EVENT(VF_UPDATE))
115
116#define _WRAPGTEQ(a, b) ((a-b) > 0)
117
118struct nvgpu_clk_notification {
119 u32 notification;
120 u64 timestamp;
121};
122
123struct nvgpu_clk_notification_queue {
124 u32 size;
125 nvgpu_atomic_t head;
126 nvgpu_atomic_t tail;
127 struct nvgpu_clk_notification *notifications;
128};
129
130struct nvgpu_clk_vf_point {
131 u16 pstates;
132 union {
133 struct {
134 u16 gpc_mhz;
135 u16 sys_mhz;
136 u16 xbar_mhz;
137 };
138 u16 mem_mhz;
139 };
140 u32 uvolt;
141 u32 uvolt_sram;
142};
143
144struct nvgpu_clk_vf_table {
145 u32 mclk_num_points;
146 struct nvgpu_clk_vf_point *mclk_points;
147 u32 gpc2clk_num_points;
148 struct nvgpu_clk_vf_point *gpc2clk_points;
149};
150#ifdef CONFIG_DEBUG_FS
151struct nvgpu_clk_arb_debug {
152 s64 switch_max;
153 s64 switch_min;
154 u64 switch_num;
155 s64 switch_avg;
156 s64 switch_std;
157};
158#endif
159
160struct nvgpu_clk_arb_target {
161 u16 mclk;
162 u16 gpc2clk;
163 u32 pstate;
164};
165
166struct nvgpu_clk_arb {
167 struct nvgpu_spinlock sessions_lock;
168 struct nvgpu_spinlock users_lock;
169
170 struct nvgpu_mutex pstate_lock;
171 struct list_head users;
172 struct list_head sessions;
173 struct llist_head requests;
174
175 struct gk20a *g;
176 int status;
177
178 struct nvgpu_clk_arb_target actual_pool[2];
179 struct nvgpu_clk_arb_target *actual;
180
181 u16 gpc2clk_default_mhz;
182 u16 mclk_default_mhz;
183 u32 voltuv_actual;
184
185 u16 gpc2clk_min, gpc2clk_max;
186 u16 mclk_min, mclk_max;
187
188 struct work_struct update_fn_work;
189 struct workqueue_struct *update_work_queue;
190 struct work_struct vf_table_fn_work;
191 struct workqueue_struct *vf_table_work_queue;
192
193 struct nvgpu_cond request_wq;
194
195 struct nvgpu_clk_vf_table *current_vf_table;
196 struct nvgpu_clk_vf_table vf_table_pool[2];
197 u32 vf_table_index;
198
199 u16 *mclk_f_points;
200 nvgpu_atomic_t req_nr;
201
202 u32 mclk_f_numpoints;
203 u16 *gpc2clk_f_points;
204 u32 gpc2clk_f_numpoints;
205
206 nvgpu_atomic64_t alarm_mask;
207 struct nvgpu_clk_notification_queue notification_queue;
208
209#ifdef CONFIG_DEBUG_FS
210 struct nvgpu_clk_arb_debug debug_pool[2];
211 struct nvgpu_clk_arb_debug *debug;
212 bool debugfs_set;
213#endif
214};
215
216struct nvgpu_clk_dev {
217 struct nvgpu_clk_session *session;
218 union {
219 struct list_head link;
220 struct llist_node node;
221 };
222 struct nvgpu_cond readout_wq;
223 nvgpu_atomic_t poll_mask;
224 u16 gpc2clk_target_mhz;
225 u16 mclk_target_mhz;
226 u32 alarms_reported;
227 nvgpu_atomic_t enabled_mask;
228 struct nvgpu_clk_notification_queue queue;
229 u32 arb_queue_head;
230 struct nvgpu_ref refcount;
231};
232
233struct nvgpu_clk_session {
234 bool zombie;
235 struct gk20a *g;
236 struct nvgpu_ref refcount;
237 struct list_head link;
238 struct llist_head targets;
239
240 struct nvgpu_clk_arb_target target_pool[2];
241 struct nvgpu_clk_arb_target *target;
242};
243
244static const struct file_operations completion_dev_ops = {
245 .owner = THIS_MODULE,
246 .release = nvgpu_clk_arb_release_completion_dev,
247 .poll = nvgpu_clk_arb_poll_dev,
248};
249
250static const struct file_operations event_dev_ops = {
251 .owner = THIS_MODULE,
252 .release = nvgpu_clk_arb_release_event_dev,
253 .poll = nvgpu_clk_arb_poll_dev,
254 .read = nvgpu_clk_arb_read_event_dev,
255#ifdef CONFIG_COMPAT
256 .compat_ioctl = nvgpu_clk_arb_ioctl_event_dev,
257#endif
258 .unlocked_ioctl = nvgpu_clk_arb_ioctl_event_dev,
259};
260
261static int nvgpu_clk_notification_queue_alloc(struct gk20a *g,
262 struct nvgpu_clk_notification_queue *queue,
263 size_t events_number) {
264 queue->notifications = nvgpu_kcalloc(g, events_number,
265 sizeof(struct nvgpu_clk_notification));
266 if (!queue->notifications)
267 return -ENOMEM;
268 queue->size = events_number;
269
270 nvgpu_atomic_set(&queue->head, 0);
271 nvgpu_atomic_set(&queue->tail, 0);
272
273 return 0;
274}
275
276static void nvgpu_clk_notification_queue_free(struct gk20a *g,
277 struct nvgpu_clk_notification_queue *queue) {
278 nvgpu_kfree(g, queue->notifications);
279 queue->size = 0;
280 nvgpu_atomic_set(&queue->head, 0);
281 nvgpu_atomic_set(&queue->tail, 0);
282}
283
284int nvgpu_clk_arb_init_arbiter(struct gk20a *g)
285{
286 struct nvgpu_clk_arb *arb;
287 u16 default_mhz;
288 int err;
289 int index;
290 struct nvgpu_clk_vf_table *table;
291
292 gk20a_dbg_fn("");
293
294 if (!g->ops.clk_arb.get_arbiter_clk_domains)
295 return 0;
296
297 arb = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_arb));
298 if (!arb)
299 return -ENOMEM;
300
301 err = nvgpu_mutex_init(&arb->pstate_lock);
302 if (err)
303 goto mutex_fail;
304 nvgpu_spinlock_init(&arb->sessions_lock);
305 nvgpu_spinlock_init(&arb->users_lock);
306
307 arb->mclk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16));
308 if (!arb->mclk_f_points) {
309 err = -ENOMEM;
310 goto init_fail;
311 }
312
313 arb->gpc2clk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16));
314 if (!arb->gpc2clk_f_points) {
315 err = -ENOMEM;
316 goto init_fail;
317 }
318
319 for (index = 0; index < 2; index++) {
320 table = &arb->vf_table_pool[index];
321 table->gpc2clk_num_points = MAX_F_POINTS;
322 table->mclk_num_points = MAX_F_POINTS;
323
324 table->gpc2clk_points = nvgpu_kcalloc(g, MAX_F_POINTS,
325 sizeof(struct nvgpu_clk_vf_point));
326 if (!table->gpc2clk_points) {
327 err = -ENOMEM;
328 goto init_fail;
329 }
330
331
332 table->mclk_points = nvgpu_kcalloc(g, MAX_F_POINTS,
333 sizeof(struct nvgpu_clk_vf_point));
334 if (!table->mclk_points) {
335 err = -ENOMEM;
336 goto init_fail;
337 }
338 }
339
340 g->clk_arb = arb;
341 arb->g = g;
342
343 err = g->ops.clk_arb.get_arbiter_clk_default(g,
344 CTRL_CLK_DOMAIN_MCLK, &default_mhz);
345 if (err < 0) {
346 err = -EINVAL;
347 goto init_fail;
348 }
349
350 arb->mclk_default_mhz = default_mhz;
351
352 err = g->ops.clk_arb.get_arbiter_clk_default(g,
353 CTRL_CLK_DOMAIN_GPC2CLK, &default_mhz);
354 if (err < 0) {
355 err = -EINVAL;
356 goto init_fail;
357 }
358
359 arb->gpc2clk_default_mhz = default_mhz;
360
361 arb->actual = &arb->actual_pool[0];
362
363 nvgpu_atomic_set(&arb->req_nr, 0);
364
365 nvgpu_atomic64_set(&arb->alarm_mask, 0);
366 err = nvgpu_clk_notification_queue_alloc(g, &arb->notification_queue,
367 DEFAULT_EVENT_NUMBER);
368 if (err < 0)
369 goto init_fail;
370
371 INIT_LIST_HEAD_RCU(&arb->users);
372 INIT_LIST_HEAD_RCU(&arb->sessions);
373 init_llist_head(&arb->requests);
374
375 nvgpu_cond_init(&arb->request_wq);
376 arb->vf_table_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1,
377 "vf_table_update");
378 arb->update_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1,
379 "arbiter_update");
380
381
382 INIT_WORK(&arb->vf_table_fn_work, nvgpu_clk_arb_run_vf_table_cb);
383
384 INIT_WORK(&arb->update_fn_work, nvgpu_clk_arb_run_arbiter_cb);
385
386#ifdef CONFIG_DEBUG_FS
387 arb->debug = &arb->debug_pool[0];
388
389 if (!arb->debugfs_set) {
390 if (nvgpu_clk_arb_debugfs_init(g))
391 arb->debugfs_set = true;
392 }
393#endif
394 err = clk_vf_point_cache(g);
395 if (err < 0)
396 goto init_fail;
397
398 err = nvgpu_clk_arb_update_vf_table(arb);
399 if (err < 0)
400 goto init_fail;
401 do {
402 /* Check that first run is completed */
403 nvgpu_smp_mb();
404 NVGPU_COND_WAIT_INTERRUPTIBLE(&arb->request_wq,
405 nvgpu_atomic_read(&arb->req_nr), 0);
406 } while (!nvgpu_atomic_read(&arb->req_nr));
407
408
409 return arb->status;
410
411init_fail:
412 nvgpu_kfree(g, arb->gpc2clk_f_points);
413 nvgpu_kfree(g, arb->mclk_f_points);
414
415 for (index = 0; index < 2; index++) {
416 nvgpu_kfree(g, arb->vf_table_pool[index].gpc2clk_points);
417 nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points);
418 }
419
420 nvgpu_mutex_destroy(&arb->pstate_lock);
421
422mutex_fail:
423 nvgpu_kfree(g, arb);
424
425 return err;
426}
427
428void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm)
429{
430 struct nvgpu_clk_arb *arb = g->clk_arb;
431
432 nvgpu_clk_arb_set_global_alarm(g, alarm);
433 if (arb->update_work_queue)
434 queue_work(arb->update_work_queue, &arb->update_fn_work);
435}
436
437static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm)
438{
439 struct nvgpu_clk_arb *arb = g->clk_arb;
440
441 u64 current_mask;
442 u32 refcnt;
443 u32 alarm_mask;
444 u64 new_mask;
445
446 do {
447 current_mask = nvgpu_atomic64_read(&arb->alarm_mask);
448 /* atomic operations are strong so they do not need masks */
449
450 refcnt = ((u32) (current_mask >> 32)) + 1;
451 alarm_mask = (u32) (current_mask & ~alarm);
452 new_mask = ((u64) refcnt << 32) | alarm_mask;
453
454 } while (unlikely(current_mask !=
455 (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask,
456 current_mask, new_mask)));
457}
458
459static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm)
460{
461 struct nvgpu_clk_arb *arb = g->clk_arb;
462
463 u64 current_mask;
464 u32 refcnt;
465 u32 alarm_mask;
466 u64 new_mask;
467
468 do {
469 current_mask = nvgpu_atomic64_read(&arb->alarm_mask);
470 /* atomic operations are strong so they do not need masks */
471
472 refcnt = ((u32) (current_mask >> 32)) + 1;
473 alarm_mask = (u32) (current_mask & ~0) | alarm;
474 new_mask = ((u64) refcnt << 32) | alarm_mask;
475
476 } while (unlikely(current_mask !=
477 (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask,
478 current_mask, new_mask)));
479
480 nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm);
481
482}
483
484void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g)
485{
486 struct nvgpu_clk_arb *arb = g->clk_arb;
487 int index;
488
489 if (arb) {
490 cancel_work_sync(&arb->vf_table_fn_work);
491 destroy_workqueue(arb->vf_table_work_queue);
492 arb->vf_table_work_queue = NULL;
493
494 cancel_work_sync(&arb->update_fn_work);
495 destroy_workqueue(arb->update_work_queue);
496 arb->update_work_queue = NULL;
497
498 nvgpu_kfree(g, arb->gpc2clk_f_points);
499 nvgpu_kfree(g, arb->mclk_f_points);
500
501 for (index = 0; index < 2; index++) {
502 nvgpu_kfree(g, arb->vf_table_pool[index].gpc2clk_points);
503 nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points);
504 }
505 nvgpu_mutex_destroy(&g->clk_arb->pstate_lock);
506 nvgpu_kfree(g, g->clk_arb);
507 g->clk_arb = NULL;
508 }
509}
510
511static int nvgpu_clk_arb_install_fd(struct gk20a *g,
512 struct nvgpu_clk_session *session,
513 const struct file_operations *fops,
514 struct nvgpu_clk_dev **_dev)
515{
516 struct file *file;
517 int fd;
518 int err;
519 int status;
520 char name[64];
521 struct nvgpu_clk_dev *dev;
522
523 gk20a_dbg_fn("");
524
525 dev = nvgpu_kzalloc(g, sizeof(*dev));
526 if (!dev)
527 return -ENOMEM;
528
529 status = nvgpu_clk_notification_queue_alloc(g, &dev->queue,
530 DEFAULT_EVENT_NUMBER);
531 if (status < 0) {
532 err = status;
533 goto fail;
534 }
535
536 fd = get_unused_fd_flags(O_RDWR);
537 if (fd < 0) {
538 err = fd;
539 goto fail;
540 }
541
542 snprintf(name, sizeof(name), "%s-clk-fd%d", g->name, fd);
543 file = anon_inode_getfile(name, fops, dev, O_RDWR);
544 if (IS_ERR(file)) {
545 err = PTR_ERR(file);
546 goto fail_fd;
547 }
548
549 fd_install(fd, file);
550
551 nvgpu_cond_init(&dev->readout_wq);
552
553 nvgpu_atomic_set(&dev->poll_mask, 0);
554
555 dev->session = session;
556 nvgpu_ref_init(&dev->refcount);
557
558 nvgpu_ref_get(&session->refcount);
559
560 *_dev = dev;
561
562 return fd;
563
564fail_fd:
565 put_unused_fd(fd);
566fail:
567 nvgpu_kfree(g, dev);
568
569 return err;
570}
571
572int nvgpu_clk_arb_init_session(struct gk20a *g,
573 struct nvgpu_clk_session **_session)
574{
575 struct nvgpu_clk_arb *arb = g->clk_arb;
576 struct nvgpu_clk_session *session = *(_session);
577
578 gk20a_dbg_fn("");
579
580 if (!g->ops.clk_arb.get_arbiter_clk_domains)
581 return 0;
582
583 session = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_session));
584 if (!session)
585 return -ENOMEM;
586 session->g = g;
587
588 nvgpu_ref_init(&session->refcount);
589
590 session->zombie = false;
591 session->target_pool[0].pstate = CTRL_PERF_PSTATE_P8;
592 /* make sure that the initialization of the pool is visible
593 * before the update */
594 nvgpu_smp_wmb();
595 session->target = &session->target_pool[0];
596
597 init_llist_head(&session->targets);
598
599 nvgpu_spinlock_acquire(&arb->sessions_lock);
600 list_add_tail_rcu(&session->link, &arb->sessions);
601 nvgpu_spinlock_release(&arb->sessions_lock);
602
603 *_session = session;
604
605 return 0;
606}
607
608static void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount)
609{
610 struct nvgpu_clk_dev *dev = container_of(refcount,
611 struct nvgpu_clk_dev, refcount);
612 struct nvgpu_clk_session *session = dev->session;
613
614 nvgpu_kfree(session->g, dev);
615}
616
617static void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount)
618{
619 struct nvgpu_clk_session *session = container_of(refcount,
620 struct nvgpu_clk_session, refcount);
621 struct nvgpu_clk_arb *arb = session->g->clk_arb;
622 struct gk20a *g = session->g;
623 struct nvgpu_clk_dev *dev, *tmp;
624 struct llist_node *head;
625
626 gk20a_dbg_fn("");
627
628 if (arb) {
629 nvgpu_spinlock_acquire(&arb->sessions_lock);
630 list_del_rcu(&session->link);
631 nvgpu_spinlock_release(&arb->sessions_lock);
632 }
633
634 head = llist_del_all(&session->targets);
635 llist_for_each_entry_safe(dev, tmp, head, node) {
636 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
637 }
638 synchronize_rcu();
639 nvgpu_kfree(g, session);
640}
641
642void nvgpu_clk_arb_release_session(struct gk20a *g,
643 struct nvgpu_clk_session *session)
644{
645 struct nvgpu_clk_arb *arb = g->clk_arb;
646
647 gk20a_dbg_fn("");
648
649 session->zombie = true;
650 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
651 if (arb && arb->update_work_queue)
652 queue_work(arb->update_work_queue, &arb->update_fn_work);
653}
654
655int nvgpu_clk_arb_install_event_fd(struct gk20a *g,
656 struct nvgpu_clk_session *session, int *event_fd, u32 alarm_mask)
657{
658 struct nvgpu_clk_arb *arb = g->clk_arb;
659 struct nvgpu_clk_dev *dev;
660 int fd;
661
662 gk20a_dbg_fn("");
663
664 fd = nvgpu_clk_arb_install_fd(g, session, &event_dev_ops, &dev);
665 if (fd < 0)
666 return fd;
667
668 /* TODO: alarm mask needs to be set to default value to prevent
669 * failures of legacy tests. This will be removed when sanity is
670 * updated
671 */
672 if (alarm_mask)
673 nvgpu_atomic_set(&dev->enabled_mask, alarm_mask);
674 else
675 nvgpu_atomic_set(&dev->enabled_mask, EVENT(VF_UPDATE));
676
677 dev->arb_queue_head = nvgpu_atomic_read(&arb->notification_queue.head);
678
679 nvgpu_spinlock_acquire(&arb->users_lock);
680 list_add_tail_rcu(&dev->link, &arb->users);
681 nvgpu_spinlock_release(&arb->users_lock);
682
683 *event_fd = fd;
684
685 return 0;
686}
687
688int nvgpu_clk_arb_install_request_fd(struct gk20a *g,
689 struct nvgpu_clk_session *session, int *request_fd)
690{
691 struct nvgpu_clk_dev *dev;
692 int fd;
693
694 gk20a_dbg_fn("");
695
696 fd = nvgpu_clk_arb_install_fd(g, session, &completion_dev_ops, &dev);
697 if (fd < 0)
698 return fd;
699
700 *request_fd = fd;
701
702 return 0;
703}
704
705static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb)
706{
707 struct gk20a *g = arb->g;
708 struct nvgpu_clk_vf_table *table;
709
710 u32 i, j;
711 int status = -EINVAL;
712 u32 gpc2clk_voltuv = 0, mclk_voltuv = 0;
713 u32 gpc2clk_voltuv_sram = 0, mclk_voltuv_sram = 0;
714 u16 clk_cur;
715 u32 num_points;
716
717 struct clk_set_info *p5_info, *p0_info;
718
719
720 table = NV_ACCESS_ONCE(arb->current_vf_table);
721 /* make flag visible when all data has resolved in the tables */
722 nvgpu_smp_rmb();
723
724 table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] :
725 &arb->vf_table_pool[0];
726
727 /* Get allowed memory ranges */
728 if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_GPC2CLK,
729 &arb->gpc2clk_min,
730 &arb->gpc2clk_max) < 0) {
731 nvgpu_err(g, "failed to fetch GPC2CLK range");
732 goto exit_vf_table;
733 }
734 if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_MCLK,
735 &arb->mclk_min,
736 &arb->mclk_max) < 0) {
737 nvgpu_err(g, "failed to fetch MCLK range");
738 goto exit_vf_table;
739 }
740
741 table->gpc2clk_num_points = MAX_F_POINTS;
742 table->mclk_num_points = MAX_F_POINTS;
743
744 if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_GPC2CLK,
745 &table->gpc2clk_num_points, arb->gpc2clk_f_points)) {
746 nvgpu_err(g, "failed to fetch GPC2CLK frequency points");
747 goto exit_vf_table;
748 }
749
750 if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_MCLK,
751 &table->mclk_num_points, arb->mclk_f_points)) {
752 nvgpu_err(g, "failed to fetch MCLK frequency points");
753 goto exit_vf_table;
754 }
755 if (!table->mclk_num_points || !table->gpc2clk_num_points) {
756 nvgpu_err(g, "empty queries to f points mclk %d gpc2clk %d",
757 table->mclk_num_points, table->gpc2clk_num_points);
758 status = -EINVAL;
759 goto exit_vf_table;
760 }
761
762 memset(table->mclk_points, 0,
763 table->mclk_num_points*sizeof(struct nvgpu_clk_vf_point));
764 memset(table->gpc2clk_points, 0,
765 table->gpc2clk_num_points*sizeof(struct nvgpu_clk_vf_point));
766
767 p5_info = pstate_get_clk_set_info(g,
768 CTRL_PERF_PSTATE_P5, clkwhich_mclk);
769 if (!p5_info) {
770 nvgpu_err(g, "failed to get MCLK P5 info");
771 goto exit_vf_table;
772 }
773 p0_info = pstate_get_clk_set_info(g,
774 CTRL_PERF_PSTATE_P0, clkwhich_mclk);
775 if (!p0_info) {
776 nvgpu_err(g, "failed to get MCLK P0 info");
777 goto exit_vf_table;
778 }
779
780 for (i = 0, j = 0, num_points = 0, clk_cur = 0;
781 i < table->mclk_num_points; i++) {
782
783 if ((arb->mclk_f_points[i] >= arb->mclk_min) &&
784 (arb->mclk_f_points[i] <= arb->mclk_max) &&
785 (arb->mclk_f_points[i] != clk_cur)) {
786
787 table->mclk_points[j].mem_mhz = arb->mclk_f_points[i];
788 mclk_voltuv = mclk_voltuv_sram = 0;
789
790 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK,
791 &table->mclk_points[j].mem_mhz, &mclk_voltuv,
792 CTRL_VOLT_DOMAIN_LOGIC);
793 if (status < 0) {
794 nvgpu_err(g,
795 "failed to get MCLK LOGIC voltage");
796 goto exit_vf_table;
797 }
798 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK,
799 &table->mclk_points[j].mem_mhz,
800 &mclk_voltuv_sram,
801 CTRL_VOLT_DOMAIN_SRAM);
802 if (status < 0) {
803 nvgpu_err(g, "failed to get MCLK SRAM voltage");
804 goto exit_vf_table;
805 }
806
807 table->mclk_points[j].uvolt = mclk_voltuv;
808 table->mclk_points[j].uvolt_sram = mclk_voltuv_sram;
809 clk_cur = table->mclk_points[j].mem_mhz;
810
811 if ((clk_cur >= p5_info->min_mhz) &&
812 (clk_cur <= p5_info->max_mhz))
813 VF_POINT_SET_PSTATE_SUPPORTED(
814 &table->mclk_points[j],
815 CTRL_PERF_PSTATE_P5);
816 if ((clk_cur >= p0_info->min_mhz) &&
817 (clk_cur <= p0_info->max_mhz))
818 VF_POINT_SET_PSTATE_SUPPORTED(
819 &table->mclk_points[j],
820 CTRL_PERF_PSTATE_P0);
821
822 j++;
823 num_points++;
824
825 }
826 }
827 table->mclk_num_points = num_points;
828
829 p5_info = pstate_get_clk_set_info(g,
830 CTRL_PERF_PSTATE_P5, clkwhich_gpc2clk);
831 if (!p5_info) {
832 status = -EINVAL;
833 nvgpu_err(g, "failed to get GPC2CLK P5 info");
834 goto exit_vf_table;
835 }
836
837 p0_info = pstate_get_clk_set_info(g,
838 CTRL_PERF_PSTATE_P0, clkwhich_gpc2clk);
839 if (!p0_info) {
840 status = -EINVAL;
841 nvgpu_err(g, "failed to get GPC2CLK P0 info");
842 goto exit_vf_table;
843 }
844
845 /* GPC2CLK needs to be checked in two passes. The first determines the
846 * relationships between GPC2CLK, SYS2CLK and XBAR2CLK, while the
847 * second verifies that the clocks minimum is satisfied and sets
848 * the voltages
849 */
850 for (i = 0, j = 0, num_points = 0, clk_cur = 0;
851 i < table->gpc2clk_num_points; i++) {
852 struct set_fll_clk setfllclk;
853
854 if ((arb->gpc2clk_f_points[i] >= arb->gpc2clk_min) &&
855 (arb->gpc2clk_f_points[i] <= arb->gpc2clk_max) &&
856 (arb->gpc2clk_f_points[i] != clk_cur)) {
857
858 table->gpc2clk_points[j].gpc_mhz =
859 arb->gpc2clk_f_points[i];
860 setfllclk.gpc2clkmhz = arb->gpc2clk_f_points[i];
861 status = clk_get_fll_clks(g, &setfllclk);
862 if (status < 0) {
863 nvgpu_err(g,
864 "failed to get GPC2CLK slave clocks");
865 goto exit_vf_table;
866 }
867
868 table->gpc2clk_points[j].sys_mhz =
869 setfllclk.sys2clkmhz;
870 table->gpc2clk_points[j].xbar_mhz =
871 setfllclk.xbar2clkmhz;
872
873 clk_cur = table->gpc2clk_points[j].gpc_mhz;
874
875 if ((clk_cur >= p5_info->min_mhz) &&
876 (clk_cur <= p5_info->max_mhz))
877 VF_POINT_SET_PSTATE_SUPPORTED(
878 &table->gpc2clk_points[j],
879 CTRL_PERF_PSTATE_P5);
880 if ((clk_cur >= p0_info->min_mhz) &&
881 (clk_cur <= p0_info->max_mhz))
882 VF_POINT_SET_PSTATE_SUPPORTED(
883 &table->gpc2clk_points[j],
884 CTRL_PERF_PSTATE_P0);
885
886 j++;
887 num_points++;
888 }
889 }
890 table->gpc2clk_num_points = num_points;
891
892 /* Second pass */
893 for (i = 0, j = 0; i < table->gpc2clk_num_points; i++) {
894
895 u16 alt_gpc2clk = table->gpc2clk_points[i].gpc_mhz;
896 gpc2clk_voltuv = gpc2clk_voltuv_sram = 0;
897
898 /* Check sysclk */
899 p5_info = pstate_get_clk_set_info(g,
900 VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]),
901 clkwhich_sys2clk);
902 if (!p5_info) {
903 status = -EINVAL;
904 nvgpu_err(g, "failed to get SYS2CLK P5 info");
905 goto exit_vf_table;
906 }
907
908 /* sys2clk below clk min, need to find correct clock */
909 if (table->gpc2clk_points[i].sys_mhz < p5_info->min_mhz) {
910 for (j = i + 1; j < table->gpc2clk_num_points; j++) {
911
912 if (table->gpc2clk_points[j].sys_mhz >=
913 p5_info->min_mhz) {
914
915
916 table->gpc2clk_points[i].sys_mhz =
917 p5_info->min_mhz;
918
919 alt_gpc2clk = alt_gpc2clk <
920 table->gpc2clk_points[j].
921 gpc_mhz ?
922 table->gpc2clk_points[j].
923 gpc_mhz:
924 alt_gpc2clk;
925 break;
926 }
927 }
928 /* no VF exists that satisfies condition */
929 if (j == table->gpc2clk_num_points) {
930 nvgpu_err(g, "NO SYS2CLK VF point possible");
931 status = -EINVAL;
932 goto exit_vf_table;
933 }
934 }
935
936 /* Check xbarclk */
937 p5_info = pstate_get_clk_set_info(g,
938 VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]),
939 clkwhich_xbar2clk);
940 if (!p5_info) {
941 status = -EINVAL;
942 nvgpu_err(g, "failed to get SYS2CLK P5 info");
943 goto exit_vf_table;
944 }
945
946 /* xbar2clk below clk min, need to find correct clock */
947 if (table->gpc2clk_points[i].xbar_mhz < p5_info->min_mhz) {
948 for (j = i; j < table->gpc2clk_num_points; j++) {
949 if (table->gpc2clk_points[j].xbar_mhz >=
950 p5_info->min_mhz) {
951
952 table->gpc2clk_points[i].xbar_mhz =
953 p5_info->min_mhz;
954
955 alt_gpc2clk = alt_gpc2clk <
956 table->gpc2clk_points[j].
957 gpc_mhz ?
958 table->gpc2clk_points[j].
959 gpc_mhz:
960 alt_gpc2clk;
961 break;
962 }
963 }
964 /* no VF exists that satisfies condition */
965 if (j == table->gpc2clk_num_points) {
966 status = -EINVAL;
967 nvgpu_err(g, "NO XBAR2CLK VF point possible");
968
969 goto exit_vf_table;
970 }
971 }
972
973 /* Calculate voltages */
974 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK,
975 &alt_gpc2clk, &gpc2clk_voltuv,
976 CTRL_VOLT_DOMAIN_LOGIC);
977 if (status < 0) {
978 nvgpu_err(g, "failed to get GPC2CLK LOGIC voltage");
979 goto exit_vf_table;
980 }
981
982 status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK,
983 &alt_gpc2clk,
984 &gpc2clk_voltuv_sram,
985 CTRL_VOLT_DOMAIN_SRAM);
986 if (status < 0) {
987 nvgpu_err(g, "failed to get GPC2CLK SRAM voltage");
988 goto exit_vf_table;
989 }
990
991 table->gpc2clk_points[i].uvolt = gpc2clk_voltuv;
992 table->gpc2clk_points[i].uvolt_sram = gpc2clk_voltuv_sram;
993 }
994
995 /* make table visible when all data has resolved in the tables */
996 nvgpu_smp_wmb();
997 xchg(&arb->current_vf_table, table);
998
999exit_vf_table:
1000
1001 if (status < 0)
1002 nvgpu_clk_arb_set_global_alarm(g,
1003 EVENT(ALARM_VF_TABLE_UPDATE_FAILED));
1004 if (arb->update_work_queue)
1005 queue_work(arb->update_work_queue, &arb->update_fn_work);
1006
1007 return status;
1008}
1009
1010void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g)
1011{
1012 struct nvgpu_clk_arb *arb = g->clk_arb;
1013 if (arb->vf_table_work_queue)
1014 queue_work(arb->vf_table_work_queue, &arb->vf_table_fn_work);
1015}
1016
1017static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work)
1018{
1019 struct nvgpu_clk_arb *arb =
1020 container_of(work, struct nvgpu_clk_arb, vf_table_fn_work);
1021 struct gk20a *g = arb->g;
1022 u32 err;
1023
1024 /* get latest vf curve from pmu */
1025 err = clk_vf_point_cache(g);
1026 if (err) {
1027 nvgpu_err(g, "failed to cache VF table");
1028 nvgpu_clk_arb_set_global_alarm(g,
1029 EVENT(ALARM_VF_TABLE_UPDATE_FAILED));
1030 if (arb->update_work_queue)
1031 queue_work(arb->update_work_queue,
1032 &arb->update_fn_work);
1033
1034 return;
1035 }
1036 nvgpu_clk_arb_update_vf_table(arb);
1037}
1038
1039static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work)
1040{
1041 struct nvgpu_clk_arb *arb =
1042 container_of(work, struct nvgpu_clk_arb, update_fn_work);
1043 struct nvgpu_clk_session *session;
1044 struct nvgpu_clk_dev *dev;
1045 struct nvgpu_clk_dev *tmp;
1046 struct nvgpu_clk_arb_target *target, *actual;
1047 struct gk20a *g = arb->g;
1048 struct llist_node *head;
1049
1050 u32 pstate = VF_POINT_INVALID_PSTATE;
1051 u32 voltuv, voltuv_sram;
1052 bool mclk_set, gpc2clk_set;
1053 u32 nuvmin, nuvmin_sram;
1054
1055 u32 alarms_notified = 0;
1056 u32 current_alarm;
1057 int status = 0;
1058
1059 /* Temporary variables for checking target frequency */
1060 u16 gpc2clk_target, sys2clk_target, xbar2clk_target, mclk_target;
1061 u16 gpc2clk_session_target, mclk_session_target;
1062
1063#ifdef CONFIG_DEBUG_FS
1064 u64 t0, t1;
1065 struct nvgpu_clk_arb_debug *debug;
1066
1067#endif
1068
1069 gk20a_dbg_fn("");
1070
1071 /* bail out if gpu is down */
1072 if (nvgpu_atomic64_read(&arb->alarm_mask) & EVENT(ALARM_GPU_LOST))
1073 goto exit_arb;
1074
1075#ifdef CONFIG_DEBUG_FS
1076 g->ops.bus.read_ptimer(g, &t0);
1077#endif
1078
1079 /* Only one arbiter should be running */
1080 gpc2clk_target = 0;
1081 mclk_target = 0;
1082
1083 rcu_read_lock();
1084 list_for_each_entry_rcu(session, &arb->sessions, link) {
1085 if (!session->zombie) {
1086 mclk_set = false;
1087 gpc2clk_set = false;
1088 target = NV_ACCESS_ONCE(session->target) ==
1089 &session->target_pool[0] ?
1090 &session->target_pool[1] :
1091 &session->target_pool[0];
1092 /* Do not reorder pointer */
1093 nvgpu_smp_rmb();
1094 head = llist_del_all(&session->targets);
1095 if (head) {
1096
1097 /* Copy over state */
1098 target->mclk = session->target->mclk;
1099 target->gpc2clk = session->target->gpc2clk;
1100 /* Query the latest committed request */
1101 llist_for_each_entry_safe(dev, tmp, head,
1102 node) {
1103 if (!mclk_set && dev->mclk_target_mhz) {
1104 target->mclk =
1105 dev->mclk_target_mhz;
1106 mclk_set = true;
1107 }
1108 if (!gpc2clk_set &&
1109 dev->gpc2clk_target_mhz) {
1110 target->gpc2clk =
1111 dev->gpc2clk_target_mhz;
1112 gpc2clk_set = true;
1113 }
1114 nvgpu_ref_get(&dev->refcount);
1115 llist_add(&dev->node, &arb->requests);
1116 }
1117 /* Ensure target is updated before ptr sawp */
1118 nvgpu_smp_wmb();
1119 xchg(&session->target, target);
1120 }
1121
1122 mclk_target = mclk_target > session->target->mclk ?
1123 mclk_target : session->target->mclk;
1124
1125 gpc2clk_target =
1126 gpc2clk_target > session->target->gpc2clk ?
1127 gpc2clk_target : session->target->gpc2clk;
1128 }
1129 }
1130 rcu_read_unlock();
1131
1132 gpc2clk_target = (gpc2clk_target > 0) ? gpc2clk_target :
1133 arb->gpc2clk_default_mhz;
1134
1135 if (gpc2clk_target < arb->gpc2clk_min)
1136 gpc2clk_target = arb->gpc2clk_min;
1137
1138 if (gpc2clk_target > arb->gpc2clk_max)
1139 gpc2clk_target = arb->gpc2clk_max;
1140
1141 mclk_target = (mclk_target > 0) ? mclk_target:
1142 arb->mclk_default_mhz;
1143
1144 if (mclk_target < arb->mclk_min)
1145 mclk_target = arb->mclk_min;
1146
1147 if (mclk_target > arb->mclk_max)
1148 mclk_target = arb->mclk_max;
1149
1150 sys2clk_target = 0;
1151 xbar2clk_target = 0;
1152
1153 gpc2clk_session_target = gpc2clk_target;
1154 mclk_session_target = mclk_target;
1155
1156 /* Query the table for the closest vf point to program */
1157 pstate = nvgpu_clk_arb_find_vf_point(arb, &gpc2clk_target,
1158 &sys2clk_target, &xbar2clk_target, &mclk_target, &voltuv,
1159 &voltuv_sram, &nuvmin, &nuvmin_sram);
1160
1161 if (pstate == VF_POINT_INVALID_PSTATE) {
1162 arb->status = -EINVAL;
1163 /* make status visible */
1164 nvgpu_smp_mb();
1165 goto exit_arb;
1166 }
1167
1168 if ((gpc2clk_target < gpc2clk_session_target) ||
1169 (mclk_target < mclk_session_target))
1170 nvgpu_clk_arb_set_global_alarm(g,
1171 EVENT(ALARM_TARGET_VF_NOT_POSSIBLE));
1172
1173 if ((arb->actual->gpc2clk == gpc2clk_target) &&
1174 (arb->actual->mclk == mclk_target) &&
1175 (arb->voltuv_actual == voltuv)) {
1176 goto exit_arb;
1177 }
1178
1179 /* Program clocks */
1180 /* A change in both mclk of gpc2clk may require a change in voltage */
1181
1182 nvgpu_mutex_acquire(&arb->pstate_lock);
1183 status = nvgpu_lpwr_disable_pg(g, false);
1184
1185 status = clk_pmu_freq_controller_load(g, false,
1186 CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL);
1187 if (status < 0) {
1188 arb->status = status;
1189 nvgpu_mutex_release(&arb->pstate_lock);
1190
1191 /* make status visible */
1192 nvgpu_smp_mb();
1193 goto exit_arb;
1194 }
1195 status = volt_set_noiseaware_vmin(g, nuvmin, nuvmin_sram);
1196 if (status < 0) {
1197 arb->status = status;
1198 nvgpu_mutex_release(&arb->pstate_lock);
1199
1200 /* make status visible */
1201 nvgpu_smp_mb();
1202 goto exit_arb;
1203 }
1204
1205 status = nvgpu_clk_arb_change_vf_point(g, gpc2clk_target,
1206 sys2clk_target, xbar2clk_target, mclk_target, voltuv,
1207 voltuv_sram);
1208 if (status < 0) {
1209 arb->status = status;
1210 nvgpu_mutex_release(&arb->pstate_lock);
1211
1212 /* make status visible */
1213 nvgpu_smp_mb();
1214 goto exit_arb;
1215 }
1216
1217 status = clk_pmu_freq_controller_load(g, true,
1218 CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL);
1219 if (status < 0) {
1220 arb->status = status;
1221 nvgpu_mutex_release(&arb->pstate_lock);
1222
1223 /* make status visible */
1224 nvgpu_smp_mb();
1225 goto exit_arb;
1226 }
1227
1228 status = nvgpu_lwpr_mclk_change(g, pstate);
1229 if (status < 0) {
1230 arb->status = status;
1231 nvgpu_mutex_release(&arb->pstate_lock);
1232
1233 /* make status visible */
1234 nvgpu_smp_mb();
1235 goto exit_arb;
1236 }
1237
1238 actual = NV_ACCESS_ONCE(arb->actual) == &arb->actual_pool[0] ?
1239 &arb->actual_pool[1] : &arb->actual_pool[0];
1240
1241 /* do not reorder this pointer */
1242 nvgpu_smp_rmb();
1243 actual->gpc2clk = gpc2clk_target;
1244 actual->mclk = mclk_target;
1245 arb->voltuv_actual = voltuv;
1246 actual->pstate = pstate;
1247 arb->status = status;
1248
1249 /* Make changes visible to other threads */
1250 nvgpu_smp_wmb();
1251 xchg(&arb->actual, actual);
1252
1253 status = nvgpu_lpwr_enable_pg(g, false);
1254 if (status < 0) {
1255 arb->status = status;
1256 nvgpu_mutex_release(&arb->pstate_lock);
1257
1258 /* make status visible */
1259 nvgpu_smp_mb();
1260 goto exit_arb;
1261 }
1262
1263 /* status must be visible before atomic inc */
1264 nvgpu_smp_wmb();
1265 nvgpu_atomic_inc(&arb->req_nr);
1266
1267 /* Unlock pstate change for PG */
1268 nvgpu_mutex_release(&arb->pstate_lock);
1269
1270 /* VF Update complete */
1271 nvgpu_clk_arb_set_global_alarm(g, EVENT(VF_UPDATE));
1272
1273 nvgpu_cond_signal_interruptible(&arb->request_wq);
1274
1275#ifdef CONFIG_DEBUG_FS
1276 g->ops.bus.read_ptimer(g, &t1);
1277
1278 debug = arb->debug == &arb->debug_pool[0] ?
1279 &arb->debug_pool[1] : &arb->debug_pool[0];
1280
1281 memcpy(debug, arb->debug, sizeof(arb->debug_pool[0]));
1282 debug->switch_num++;
1283
1284 if (debug->switch_num == 1) {
1285 debug->switch_max = debug->switch_min =
1286 debug->switch_avg = (t1-t0)/1000;
1287 debug->switch_std = 0;
1288 } else {
1289 s64 prev_avg;
1290 s64 curr = (t1-t0)/1000;
1291
1292 debug->switch_max = curr > debug->switch_max ?
1293 curr : debug->switch_max;
1294 debug->switch_min = debug->switch_min ?
1295 (curr < debug->switch_min ?
1296 curr : debug->switch_min) : curr;
1297 prev_avg = debug->switch_avg;
1298 debug->switch_avg = (curr +
1299 (debug->switch_avg * (debug->switch_num-1))) /
1300 debug->switch_num;
1301 debug->switch_std +=
1302 (curr - debug->switch_avg) * (curr - prev_avg);
1303 }
1304 /* commit changes before exchanging debug pointer */
1305 nvgpu_smp_wmb();
1306 xchg(&arb->debug, debug);
1307#endif
1308
1309exit_arb:
1310 if (status < 0) {
1311 nvgpu_err(g, "Error in arbiter update");
1312 nvgpu_clk_arb_set_global_alarm(g,
1313 EVENT(ALARM_CLOCK_ARBITER_FAILED));
1314 }
1315
1316 current_alarm = (u32) nvgpu_atomic64_read(&arb->alarm_mask);
1317 /* notify completion for all requests */
1318 head = llist_del_all(&arb->requests);
1319 llist_for_each_entry_safe(dev, tmp, head, node) {
1320 nvgpu_atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM);
1321 nvgpu_cond_signal_interruptible(&dev->readout_wq);
1322 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1323 }
1324
1325 nvgpu_atomic_set(&arb->notification_queue.head,
1326 nvgpu_atomic_read(&arb->notification_queue.tail));
1327 /* notify event for all users */
1328 rcu_read_lock();
1329 list_for_each_entry_rcu(dev, &arb->users, link) {
1330 alarms_notified |=
1331 nvgpu_clk_arb_notify(dev, arb->actual, current_alarm);
1332 }
1333 rcu_read_unlock();
1334
1335 /* clear alarms */
1336 nvgpu_clk_arb_clear_global_alarm(g, alarms_notified &
1337 ~EVENT(ALARM_GPU_LOST));
1338}
1339
1340static void nvgpu_clk_arb_queue_notification(struct gk20a *g,
1341 struct nvgpu_clk_notification_queue *queue,
1342 u32 alarm_mask) {
1343
1344 u32 queue_index;
1345 u64 timestamp;
1346
1347 queue_index = (nvgpu_atomic_inc_return(&queue->tail)) % queue->size;
1348 /* get current timestamp */
1349 timestamp = (u64) sched_clock();
1350
1351 queue->notifications[queue_index].timestamp = timestamp;
1352 queue->notifications[queue_index].notification = alarm_mask;
1353
1354}
1355
1356static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev,
1357 struct nvgpu_clk_arb_target *target,
1358 u32 alarm) {
1359
1360 struct nvgpu_clk_session *session = dev->session;
1361 struct nvgpu_clk_arb *arb = session->g->clk_arb;
1362 struct nvgpu_clk_notification *notification;
1363
1364 u32 queue_alarm_mask = 0;
1365 u32 enabled_mask = 0;
1366 u32 new_alarms_reported = 0;
1367 u32 poll_mask = 0;
1368 u32 tail, head;
1369 u32 queue_index;
1370 size_t size;
1371 int index;
1372
1373 enabled_mask = nvgpu_atomic_read(&dev->enabled_mask);
1374 size = arb->notification_queue.size;
1375
1376 /* queue global arbiter notifications in buffer */
1377 do {
1378 tail = nvgpu_atomic_read(&arb->notification_queue.tail);
1379 /* copy items to the queue */
1380 queue_index = nvgpu_atomic_read(&dev->queue.tail);
1381 head = dev->arb_queue_head;
1382 head = (tail - head) < arb->notification_queue.size ?
1383 head : tail - arb->notification_queue.size;
1384
1385 for (index = head; _WRAPGTEQ(tail, index); index++) {
1386 u32 alarm_detected;
1387
1388 notification = &arb->notification_queue.
1389 notifications[(index+1) % size];
1390 alarm_detected =
1391 NV_ACCESS_ONCE(notification->notification);
1392
1393 if (!(enabled_mask & alarm_detected))
1394 continue;
1395
1396 queue_index++;
1397 dev->queue.notifications[
1398 queue_index % dev->queue.size].timestamp =
1399 NV_ACCESS_ONCE(notification->timestamp);
1400
1401 dev->queue.notifications[
1402 queue_index % dev->queue.size].notification =
1403 alarm_detected;
1404
1405 queue_alarm_mask |= alarm_detected;
1406 }
1407 } while (unlikely(nvgpu_atomic_read(&arb->notification_queue.tail) !=
1408 (int)tail));
1409
1410 nvgpu_atomic_set(&dev->queue.tail, queue_index);
1411 /* update the last notification we processed from global queue */
1412
1413 dev->arb_queue_head = tail;
1414
1415 /* Check if current session targets are met */
1416 if (enabled_mask & EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)) {
1417 if ((target->gpc2clk < session->target->gpc2clk)
1418 || (target->mclk < session->target->mclk)) {
1419
1420 poll_mask |= (POLLIN | POLLPRI);
1421 nvgpu_clk_arb_queue_notification(arb->g, &dev->queue,
1422 EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE));
1423 }
1424 }
1425
1426 /* Check if there is a new VF update */
1427 if (queue_alarm_mask & EVENT(VF_UPDATE))
1428 poll_mask |= (POLLIN | POLLRDNORM);
1429
1430 /* Notify sticky alarms that were not reported on previous run*/
1431 new_alarms_reported = (queue_alarm_mask |
1432 (alarm & ~dev->alarms_reported & queue_alarm_mask));
1433
1434 if (new_alarms_reported & ~LOCAL_ALARM_MASK) {
1435 /* check that we are not re-reporting */
1436 if (new_alarms_reported & EVENT(ALARM_GPU_LOST))
1437 poll_mask |= POLLHUP;
1438
1439 poll_mask |= (POLLIN | POLLPRI);
1440 /* On next run do not report global alarms that were already
1441 * reported, but report SHUTDOWN always */
1442 dev->alarms_reported = new_alarms_reported & ~LOCAL_ALARM_MASK &
1443 ~EVENT(ALARM_GPU_LOST);
1444 }
1445
1446 if (poll_mask) {
1447 nvgpu_atomic_set(&dev->poll_mask, poll_mask);
1448 nvgpu_cond_broadcast_interruptible(&dev->readout_wq);
1449 }
1450
1451 return new_alarms_reported;
1452}
1453
1454static int nvgpu_clk_arb_set_event_filter(struct nvgpu_clk_dev *dev,
1455 struct nvgpu_gpu_set_event_filter_args *args)
1456{
1457 u32 mask;
1458
1459 gk20a_dbg(gpu_dbg_fn, "");
1460
1461 if (args->flags)
1462 return -EINVAL;
1463
1464 if (args->size != 1)
1465 return -EINVAL;
1466
1467 if (copy_from_user(&mask, (void __user *) args->buffer,
1468 args->size * sizeof(u32)))
1469 return -EFAULT;
1470
1471 /* update alarm mask */
1472 nvgpu_atomic_set(&dev->enabled_mask, mask);
1473
1474 return 0;
1475}
1476
1477static long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd,
1478 unsigned long arg)
1479{
1480 struct nvgpu_clk_dev *dev = filp->private_data;
1481 struct gk20a *g = dev->session->g;
1482 u8 buf[NVGPU_EVENT_IOCTL_MAX_ARG_SIZE];
1483 int err = 0;
1484
1485 gk20a_dbg(gpu_dbg_fn, "nr=%d", _IOC_NR(cmd));
1486
1487 if ((_IOC_TYPE(cmd) != NVGPU_EVENT_IOCTL_MAGIC) || (_IOC_NR(cmd) == 0)
1488 || (_IOC_NR(cmd) > NVGPU_EVENT_IOCTL_LAST))
1489 return -EINVAL;
1490
1491 BUG_ON(_IOC_SIZE(cmd) > NVGPU_EVENT_IOCTL_MAX_ARG_SIZE);
1492
1493 memset(buf, 0, sizeof(buf));
1494 if (_IOC_DIR(cmd) & _IOC_WRITE) {
1495 if (copy_from_user(buf, (void __user *) arg, _IOC_SIZE(cmd)))
1496 return -EFAULT;
1497 }
1498
1499 switch (cmd) {
1500 case NVGPU_EVENT_IOCTL_SET_FILTER:
1501 err = nvgpu_clk_arb_set_event_filter(dev,
1502 (struct nvgpu_gpu_set_event_filter_args *)buf);
1503 break;
1504 default:
1505 nvgpu_warn(g, "unrecognized event ioctl cmd: 0x%x", cmd);
1506 err = -ENOTTY;
1507 }
1508
1509 if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
1510 err = copy_to_user((void __user *) arg, buf, _IOC_SIZE(cmd));
1511
1512 return err;
1513}
1514
1515int nvgpu_clk_arb_commit_request_fd(struct gk20a *g,
1516 struct nvgpu_clk_session *session, int request_fd)
1517{
1518 struct nvgpu_clk_arb *arb = g->clk_arb;
1519 struct nvgpu_clk_dev *dev;
1520 struct fd fd;
1521 int err = 0;
1522
1523 gk20a_dbg_fn("");
1524
1525 fd = fdget(request_fd);
1526 if (!fd.file)
1527 return -EINVAL;
1528
1529 if (fd.file->f_op != &completion_dev_ops) {
1530 err = -EINVAL;
1531 goto fdput_fd;
1532 }
1533
1534 dev = (struct nvgpu_clk_dev *) fd.file->private_data;
1535
1536 if (!dev || dev->session != session) {
1537 err = -EINVAL;
1538 goto fdput_fd;
1539 }
1540 nvgpu_ref_get(&dev->refcount);
1541 llist_add(&dev->node, &session->targets);
1542 if (arb->update_work_queue)
1543 queue_work(arb->update_work_queue, &arb->update_fn_work);
1544
1545fdput_fd:
1546 fdput(fd);
1547 return err;
1548}
1549
1550static inline u32 __pending_event(struct nvgpu_clk_dev *dev,
1551 struct nvgpu_gpu_event_info *info) {
1552
1553 u32 tail, head;
1554 u32 events = 0;
1555 struct nvgpu_clk_notification *p_notif;
1556
1557 tail = nvgpu_atomic_read(&dev->queue.tail);
1558 head = nvgpu_atomic_read(&dev->queue.head);
1559
1560 head = (tail - head) < dev->queue.size ? head : tail - dev->queue.size;
1561
1562 if (_WRAPGTEQ(tail, head) && info) {
1563 head++;
1564 p_notif = &dev->queue.notifications[head % dev->queue.size];
1565 events |= p_notif->notification;
1566 info->event_id = ffs(events) - 1;
1567 info->timestamp = p_notif->timestamp;
1568 nvgpu_atomic_set(&dev->queue.head, head);
1569 }
1570
1571 return events;
1572}
1573
1574static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf,
1575 size_t size, loff_t *off)
1576{
1577 struct nvgpu_clk_dev *dev = filp->private_data;
1578 struct nvgpu_gpu_event_info info;
1579 ssize_t err;
1580
1581 gk20a_dbg_fn("filp=%p, buf=%p, size=%zu", filp, buf, size);
1582
1583 if ((size - *off) < sizeof(info))
1584 return 0;
1585
1586 memset(&info, 0, sizeof(info));
1587 /* Get the oldest event from the queue */
1588 while (!__pending_event(dev, &info)) {
1589 if (filp->f_flags & O_NONBLOCK)
1590 return -EAGAIN;
1591 err = NVGPU_COND_WAIT_INTERRUPTIBLE(&dev->readout_wq,
1592 __pending_event(dev, &info), 0);
1593 if (err)
1594 return err;
1595 if (info.timestamp)
1596 break;
1597 }
1598
1599 if (copy_to_user(buf + *off, &info, sizeof(info)))
1600 return -EFAULT;
1601
1602 return sizeof(info);
1603}
1604
1605static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait)
1606{
1607 struct nvgpu_clk_dev *dev = filp->private_data;
1608
1609 gk20a_dbg_fn("");
1610
1611 poll_wait(filp, &dev->readout_wq.wq, wait);
1612 return nvgpu_atomic_xchg(&dev->poll_mask, 0);
1613}
1614
1615static int nvgpu_clk_arb_release_completion_dev(struct inode *inode,
1616 struct file *filp)
1617{
1618 struct nvgpu_clk_dev *dev = filp->private_data;
1619 struct nvgpu_clk_session *session = dev->session;
1620
1621
1622 gk20a_dbg_fn("");
1623
1624 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
1625 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1626 return 0;
1627}
1628
1629static int nvgpu_clk_arb_release_event_dev(struct inode *inode,
1630 struct file *filp)
1631{
1632 struct nvgpu_clk_dev *dev = filp->private_data;
1633 struct nvgpu_clk_session *session = dev->session;
1634 struct nvgpu_clk_arb *arb;
1635
1636 arb = session->g->clk_arb;
1637
1638 gk20a_dbg_fn("");
1639
1640 if (arb) {
1641 nvgpu_spinlock_acquire(&arb->users_lock);
1642 list_del_rcu(&dev->link);
1643 nvgpu_spinlock_release(&arb->users_lock);
1644 nvgpu_clk_notification_queue_free(arb->g, &dev->queue);
1645 }
1646
1647 synchronize_rcu();
1648 nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
1649 nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
1650
1651 return 0;
1652}
1653
1654int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session,
1655 int request_fd, u32 api_domain, u16 target_mhz)
1656{
1657 struct nvgpu_clk_dev *dev;
1658 struct fd fd;
1659 int err = 0;
1660
1661 gk20a_dbg_fn("domain=0x%08x target_mhz=%u", api_domain, target_mhz);
1662
1663 fd = fdget(request_fd);
1664 if (!fd.file)
1665 return -EINVAL;
1666
1667 if (fd.file->f_op != &completion_dev_ops) {
1668 err = -EINVAL;
1669 goto fdput_fd;
1670 }
1671
1672 dev = fd.file->private_data;
1673 if (!dev || dev->session != session) {
1674 err = -EINVAL;
1675 goto fdput_fd;
1676 }
1677
1678 switch (api_domain) {
1679 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1680 dev->mclk_target_mhz = target_mhz;
1681 break;
1682
1683 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1684 dev->gpc2clk_target_mhz = target_mhz * 2ULL;
1685 break;
1686
1687 default:
1688 err = -EINVAL;
1689 }
1690
1691fdput_fd:
1692 fdput(fd);
1693 return err;
1694}
1695
1696int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session,
1697 u32 api_domain, u16 *freq_mhz)
1698{
1699 int err = 0;
1700 struct nvgpu_clk_arb_target *target;
1701
1702 do {
1703 target = NV_ACCESS_ONCE(session->target);
1704 /* no reordering of this pointer */
1705 nvgpu_smp_rmb();
1706
1707 switch (api_domain) {
1708 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1709 *freq_mhz = target->mclk;
1710 break;
1711
1712 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1713 *freq_mhz = target->gpc2clk / 2ULL;
1714 break;
1715
1716 default:
1717 *freq_mhz = 0;
1718 err = -EINVAL;
1719 }
1720 } while (target != NV_ACCESS_ONCE(session->target));
1721 return err;
1722}
1723
1724int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g,
1725 u32 api_domain, u16 *freq_mhz)
1726{
1727 struct nvgpu_clk_arb *arb = g->clk_arb;
1728 int err = 0;
1729 struct nvgpu_clk_arb_target *actual;
1730
1731 do {
1732 actual = NV_ACCESS_ONCE(arb->actual);
1733 /* no reordering of this pointer */
1734 nvgpu_smp_rmb();
1735
1736 switch (api_domain) {
1737 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1738 *freq_mhz = actual->mclk;
1739 break;
1740
1741 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1742 *freq_mhz = actual->gpc2clk / 2ULL;
1743 break;
1744
1745 default:
1746 *freq_mhz = 0;
1747 err = -EINVAL;
1748 }
1749 } while (actual != NV_ACCESS_ONCE(arb->actual));
1750 return err;
1751}
1752
1753int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g,
1754 u32 api_domain, u16 *freq_mhz)
1755{
1756 switch(api_domain) {
1757 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1758 *freq_mhz = g->ops.clk.measure_freq(g, CTRL_CLK_DOMAIN_MCLK) /
1759 1000000ULL;
1760 return 0;
1761
1762 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1763 *freq_mhz = g->ops.clk.measure_freq(g, CTRL_CLK_DOMAIN_GPC2CLK) /
1764 2000000ULL;
1765 return 0;
1766
1767 default:
1768 return -EINVAL;
1769 }
1770}
1771
1772int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
1773 u16 *min_mhz, u16 *max_mhz)
1774{
1775 int ret;
1776
1777 switch(api_domain) {
1778 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1779 ret = g->ops.clk_arb.get_arbiter_clk_range(g,
1780 CTRL_CLK_DOMAIN_MCLK, min_mhz, max_mhz);
1781 return ret;
1782
1783 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1784 ret = g->ops.clk_arb.get_arbiter_clk_range(g,
1785 CTRL_CLK_DOMAIN_GPC2CLK, min_mhz, max_mhz);
1786 if (!ret) {
1787 *min_mhz /= 2;
1788 *max_mhz /= 2;
1789 }
1790 return ret;
1791
1792 default:
1793 return -EINVAL;
1794 }
1795}
1796
1797u32 nvgpu_clk_arb_get_arbiter_clk_domains(struct gk20a *g)
1798{
1799 u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g);
1800 u32 api_domains = 0;
1801
1802 if (clk_domains & CTRL_CLK_DOMAIN_GPC2CLK)
1803 api_domains |= BIT(NVGPU_GPU_CLK_DOMAIN_GPCCLK);
1804
1805 if (clk_domains & CTRL_CLK_DOMAIN_MCLK)
1806 api_domains |= BIT(NVGPU_GPU_CLK_DOMAIN_MCLK);
1807
1808 return api_domains;
1809}
1810
1811bool nvgpu_clk_arb_is_valid_domain(struct gk20a *g, u32 api_domain)
1812{
1813 u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g);
1814
1815 switch(api_domain) {
1816 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1817 return ((clk_domains & CTRL_CLK_DOMAIN_MCLK) != 0);
1818
1819 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1820 return ((clk_domains & CTRL_CLK_DOMAIN_GPC2CLK) != 0);
1821
1822 default:
1823 return false;
1824 }
1825}
1826
1827int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g,
1828 u32 api_domain, u32 *max_points, u16 *fpoints)
1829{
1830 int err;
1831 u32 i;
1832
1833 switch (api_domain) {
1834 case NVGPU_GPU_CLK_DOMAIN_GPCCLK:
1835 err = clk_domain_get_f_points(g, CTRL_CLK_DOMAIN_GPC2CLK,
1836 max_points, fpoints);
1837 if (err || !fpoints)
1838 return err;
1839 for (i = 0; i < *max_points; i++)
1840 fpoints[i] /= 2;
1841 return 0;
1842 case NVGPU_GPU_CLK_DOMAIN_MCLK:
1843 return clk_domain_get_f_points(g, CTRL_CLK_DOMAIN_MCLK,
1844 max_points, fpoints);
1845 default:
1846 return -EINVAL;
1847 }
1848}
1849
1850static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb,
1851 u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk,
1852 u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram)
1853{
1854 u16 gpc2clk_target, mclk_target;
1855 u32 gpc2clk_voltuv, gpc2clk_voltuv_sram;
1856 u32 mclk_voltuv, mclk_voltuv_sram;
1857 u32 pstate = VF_POINT_INVALID_PSTATE;
1858 struct nvgpu_clk_vf_table *table;
1859 u32 index, index_mclk;
1860 struct nvgpu_clk_vf_point *mclk_vf = NULL;
1861
1862 do {
1863 gpc2clk_target = *gpc2clk;
1864 mclk_target = *mclk;
1865 gpc2clk_voltuv = 0;
1866 gpc2clk_voltuv_sram = 0;
1867 mclk_voltuv = 0;
1868 mclk_voltuv_sram = 0;
1869
1870 table = NV_ACCESS_ONCE(arb->current_vf_table);
1871 /* pointer to table can be updated by callback */
1872 nvgpu_smp_rmb();
1873
1874 if (!table)
1875 continue;
1876 if ((!table->gpc2clk_num_points) || (!table->mclk_num_points)) {
1877 nvgpu_err(arb->g, "found empty table");
1878 goto find_exit;
1879 }
1880 /* First we check MCLK to find out which PSTATE we are
1881 * are requesting, and from there try to find the minimum
1882 * GPC2CLK on the same PSTATE that satisfies the request.
1883 * If no GPC2CLK can be found, then we need to up the PSTATE
1884 */
1885
1886recalculate_vf_point:
1887 for (index = 0; index < table->mclk_num_points; index++) {
1888 if (table->mclk_points[index].mem_mhz >= mclk_target) {
1889 mclk_vf = &table->mclk_points[index];
1890 break;
1891 }
1892 }
1893 if (index == table->mclk_num_points) {
1894 mclk_vf = &table->mclk_points[index-1];
1895 index = table->mclk_num_points - 1;
1896 }
1897 index_mclk = index;
1898
1899 /* round up the freq requests */
1900 for (index = 0; index < table->gpc2clk_num_points; index++) {
1901 pstate = VF_POINT_COMMON_PSTATE(
1902 &table->gpc2clk_points[index], mclk_vf);
1903
1904 if ((table->gpc2clk_points[index].gpc_mhz >=
1905 gpc2clk_target) &&
1906 (pstate != VF_POINT_INVALID_PSTATE)){
1907 gpc2clk_target =
1908 table->gpc2clk_points[index].gpc_mhz;
1909 *sys2clk =
1910 table->gpc2clk_points[index].sys_mhz;
1911 *xbar2clk =
1912 table->gpc2clk_points[index].xbar_mhz;
1913
1914 gpc2clk_voltuv =
1915 table->gpc2clk_points[index].uvolt;
1916 gpc2clk_voltuv_sram =
1917 table->gpc2clk_points[index].uvolt_sram;
1918 break;
1919 }
1920 }
1921
1922 if (index == table->gpc2clk_num_points) {
1923 pstate = VF_POINT_COMMON_PSTATE(
1924 &table->gpc2clk_points[index-1], mclk_vf);
1925 if (pstate != VF_POINT_INVALID_PSTATE) {
1926 gpc2clk_target =
1927 table->gpc2clk_points[index-1].gpc_mhz;
1928 *sys2clk =
1929 table->gpc2clk_points[index-1].sys_mhz;
1930 *xbar2clk =
1931 table->gpc2clk_points[index-1].xbar_mhz;
1932
1933 gpc2clk_voltuv =
1934 table->gpc2clk_points[index-1].uvolt;
1935 gpc2clk_voltuv_sram =
1936 table->gpc2clk_points[index-1].
1937 uvolt_sram;
1938 } else if (index_mclk >= table->mclk_num_points - 1) {
1939 /* There is no available combination of MCLK
1940 * and GPC2CLK, we need to fail this
1941 */
1942 gpc2clk_target = 0;
1943 mclk_target = 0;
1944 pstate = VF_POINT_INVALID_PSTATE;
1945 goto find_exit;
1946 } else {
1947 /* recalculate with higher PSTATE */
1948 gpc2clk_target = *gpc2clk;
1949 mclk_target = table->mclk_points[index_mclk+1].
1950 mem_mhz;
1951 goto recalculate_vf_point;
1952 }
1953 }
1954
1955 mclk_target = mclk_vf->mem_mhz;
1956 mclk_voltuv = mclk_vf->uvolt;
1957 mclk_voltuv_sram = mclk_vf->uvolt_sram;
1958
1959 } while (!table ||
1960 (NV_ACCESS_ONCE(arb->current_vf_table) != table));
1961
1962find_exit:
1963 *voltuv = gpc2clk_voltuv > mclk_voltuv ? gpc2clk_voltuv : mclk_voltuv;
1964 *voltuv_sram = gpc2clk_voltuv_sram > mclk_voltuv_sram ?
1965 gpc2clk_voltuv_sram : mclk_voltuv_sram;
1966 /* noise unaware vmin */
1967 *nuvmin = mclk_voltuv;
1968 *nuvmin_sram = mclk_voltuv_sram;
1969 *gpc2clk = gpc2clk_target < *gpc2clk ? gpc2clk_target : *gpc2clk;
1970 *mclk = mclk_target;
1971 return pstate;
1972}
1973
1974/* This function is inherently unsafe to call while arbiter is running
1975 * arbiter must be blocked before calling this function */
1976int nvgpu_clk_arb_get_current_pstate(struct gk20a *g)
1977{
1978 return NV_ACCESS_ONCE(g->clk_arb->actual->pstate);
1979}
1980
1981static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target,
1982 u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv,
1983 u32 voltuv_sram)
1984{
1985 struct set_fll_clk fllclk;
1986 struct nvgpu_clk_arb *arb = g->clk_arb;
1987 int status;
1988
1989 fllclk.gpc2clkmhz = gpc2clk_target;
1990 fllclk.sys2clkmhz = sys2clk_target;
1991 fllclk.xbar2clkmhz = xbar2clk_target;
1992
1993 fllclk.voltuv = voltuv;
1994
1995 /* if voltage ascends we do:
1996 * (1) FLL change
1997 * (2) Voltage change
1998 * (3) MCLK change
1999 * If it goes down
2000 * (1) MCLK change
2001 * (2) Voltage change
2002 * (3) FLL change
2003 */
2004
2005 /* descending */
2006 if (voltuv < arb->voltuv_actual) {
2007 status = g->ops.clk.mclk_change(g, mclk_target);
2008 if (status < 0)
2009 return status;
2010
2011 status = volt_set_voltage(g, voltuv, voltuv_sram);
2012 if (status < 0)
2013 return status;
2014
2015 status = clk_set_fll_clks(g, &fllclk);
2016 if (status < 0)
2017 return status;
2018 } else {
2019 status = clk_set_fll_clks(g, &fllclk);
2020 if (status < 0)
2021 return status;
2022
2023 status = volt_set_voltage(g, voltuv, voltuv_sram);
2024 if (status < 0)
2025 return status;
2026
2027 status = g->ops.clk.mclk_change(g, mclk_target);
2028 if (status < 0)
2029 return status;
2030 }
2031
2032 return 0;
2033}
2034
2035void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock)
2036{
2037 struct nvgpu_clk_arb *arb = g->clk_arb;
2038
2039 if (lock)
2040 nvgpu_mutex_acquire(&arb->pstate_lock);
2041 else
2042 nvgpu_mutex_release(&arb->pstate_lock);
2043}
2044
2045#ifdef CONFIG_DEBUG_FS
2046static int nvgpu_clk_arb_stats_show(struct seq_file *s, void *unused)
2047{
2048 struct gk20a *g = s->private;
2049 struct nvgpu_clk_arb *arb = g->clk_arb;
2050 struct nvgpu_clk_arb_debug *debug;
2051
2052 u64 num;
2053 s64 tmp, avg, std, max, min;
2054
2055 debug = NV_ACCESS_ONCE(arb->debug);
2056 /* Make copy of structure and ensure no reordering */
2057 nvgpu_smp_rmb();
2058 if (!debug)
2059 return -EINVAL;
2060
2061 std = debug->switch_std;
2062 avg = debug->switch_avg;
2063 max = debug->switch_max;
2064 min = debug->switch_min;
2065 num = debug->switch_num;
2066
2067 tmp = std;
2068 do_div(tmp, num);
2069 seq_printf(s, "Number of transitions: %lld\n",
2070 num);
2071 seq_printf(s, "max / min : %lld / %lld usec\n",
2072 max, min);
2073 seq_printf(s, "avg / std : %lld / %ld usec\n",
2074 avg, int_sqrt(tmp));
2075
2076 return 0;
2077}
2078
2079static int nvgpu_clk_arb_stats_open(struct inode *inode, struct file *file)
2080{
2081 return single_open(file, nvgpu_clk_arb_stats_show, inode->i_private);
2082}
2083
2084static const struct file_operations nvgpu_clk_arb_stats_fops = {
2085 .open = nvgpu_clk_arb_stats_open,
2086 .read = seq_read,
2087 .llseek = seq_lseek,
2088 .release = single_release,
2089};
2090
2091
2092static int nvgpu_clk_arb_debugfs_init(struct gk20a *g)
2093{
2094 struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
2095 struct dentry *gpu_root = l->debugfs;
2096 struct dentry *d;
2097
2098 gk20a_dbg(gpu_dbg_info, "g=%p", g);
2099
2100 d = debugfs_create_file(
2101 "arb_stats",
2102 S_IRUGO,
2103 gpu_root,
2104 g,
2105 &nvgpu_clk_arb_stats_fops);
2106 if (!d)
2107 return -ENOMEM;
2108
2109 return 0;
2110}
2111#endif
diff --git a/drivers/gpu/nvgpu/clk/clk_arb.h b/drivers/gpu/nvgpu/clk/clk_arb.h
deleted file mode 100644
index 8545e01b..00000000
--- a/drivers/gpu/nvgpu/clk/clk_arb.h
+++ /dev/null
@@ -1,80 +0,0 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef _CLK_ARB_H_
24#define _CLK_ARB_H_
25
26struct gk20a;
27struct nvgpu_clk_session;
28
29int nvgpu_clk_arb_init_arbiter(struct gk20a *g);
30
31int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
32 u16 *min_mhz, u16 *max_mhz);
33
34int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g,
35 u32 api_domain, u16 *actual_mhz);
36
37int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g,
38 u32 api_domain, u16 *effective_mhz);
39
40int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g,
41 u32 api_domain, u32 *max_points, u16 *fpoints);
42
43u32 nvgpu_clk_arb_get_arbiter_clk_domains(struct gk20a *g);
44bool nvgpu_clk_arb_is_valid_domain(struct gk20a *g, u32 api_domain);
45
46void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g);
47
48int nvgpu_clk_arb_install_session_fd(struct gk20a *g,
49 struct nvgpu_clk_session *session);
50
51int nvgpu_clk_arb_init_session(struct gk20a *g,
52 struct nvgpu_clk_session **_session);
53
54void nvgpu_clk_arb_release_session(struct gk20a *g,
55 struct nvgpu_clk_session *session);
56
57int nvgpu_clk_arb_commit_request_fd(struct gk20a *g,
58 struct nvgpu_clk_session *session, int request_fd);
59
60int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session,
61 int fd, u32 api_domain, u16 target_mhz);
62
63int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session,
64 u32 api_domain, u16 *target_mhz);
65
66int nvgpu_clk_arb_install_event_fd(struct gk20a *g,
67 struct nvgpu_clk_session *session, int *event_fd, u32 alarm_mask);
68
69int nvgpu_clk_arb_install_request_fd(struct gk20a *g,
70 struct nvgpu_clk_session *session, int *event_fd);
71
72void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g);
73
74int nvgpu_clk_arb_get_current_pstate(struct gk20a *g);
75
76void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock);
77
78void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm);
79#endif /* _CLK_ARB_H_ */
80
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-04 21:06:00 -0400