diff options
Diffstat (limited to 'drivers/gpu/nvgpu/clk/clk_arb.c')
-rw-r--r-- | drivers/gpu/nvgpu/clk/clk_arb.c | 1548 |
1 files changed, 1548 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/clk/clk_arb.c b/drivers/gpu/nvgpu/clk/clk_arb.c new file mode 100644 index 00000000..c440dc3b --- /dev/null +++ b/drivers/gpu/nvgpu/clk/clk_arb.c | |||
@@ -0,0 +1,1548 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | */ | ||
13 | |||
14 | #include "gk20a/gk20a.h" | ||
15 | |||
16 | #include <linux/cdev.h> | ||
17 | #include <linux/file.h> | ||
18 | #include <linux/anon_inodes.h> | ||
19 | #include <linux/nvgpu.h> | ||
20 | #include <linux/bitops.h> | ||
21 | #include <linux/spinlock.h> | ||
22 | #include <linux/rculist.h> | ||
23 | #include <linux/llist.h> | ||
24 | #include "clk/clk_arb.h" | ||
25 | |||
26 | |||
27 | #define MAX_F_POINTS 127 | ||
28 | |||
29 | #ifdef CONFIG_DEBUG_FS | ||
30 | static int nvgpu_clk_arb_debugfs_init(struct gk20a *g); | ||
31 | #endif | ||
32 | |||
33 | static int nvgpu_clk_arb_release_event_dev(struct inode *inode, | ||
34 | struct file *filp); | ||
35 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, | ||
36 | struct file *filp); | ||
37 | static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait); | ||
38 | |||
39 | static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work); | ||
40 | static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work); | ||
41 | static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb); | ||
42 | static void nvgpu_clk_arb_free_fd(struct kref *refcount); | ||
43 | static void nvgpu_clk_arb_free_session(struct kref *refcount); | ||
44 | static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target, | ||
45 | u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv, | ||
46 | u32 voltuv_sram); | ||
47 | static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb, | ||
48 | u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk, | ||
49 | u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram); | ||
50 | |||
51 | #define VF_POINT_INVALID_PSTATE ~0U | ||
52 | #define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b))) | ||
53 | #define VF_POINT_GET_PSTATE(a) (((a)->pstates) ?\ | ||
54 | __fls((a)->pstates) :\ | ||
55 | VF_POINT_INVALID_PSTATE) | ||
56 | #define VF_POINT_COMMON_PSTATE(a, b) (((a)->pstates & (b)->pstates) ?\ | ||
57 | __fls((a)->pstates & (b)->pstates) :\ | ||
58 | VF_POINT_INVALID_PSTATE) | ||
59 | |||
60 | struct nvgpu_clk_vf_point { | ||
61 | u16 pstates; | ||
62 | union { | ||
63 | struct { | ||
64 | u16 gpc_mhz; | ||
65 | u16 sys_mhz; | ||
66 | u16 xbar_mhz; | ||
67 | }; | ||
68 | u16 mem_mhz; | ||
69 | }; | ||
70 | u32 uvolt; | ||
71 | u32 uvolt_sram; | ||
72 | }; | ||
73 | |||
74 | struct nvgpu_clk_vf_table { | ||
75 | u32 mclk_num_points; | ||
76 | struct nvgpu_clk_vf_point *mclk_points; | ||
77 | u32 gpc2clk_num_points; | ||
78 | struct nvgpu_clk_vf_point *gpc2clk_points; | ||
79 | }; | ||
80 | #ifdef CONFIG_DEBUG_FS | ||
81 | struct nvgpu_clk_arb_debug { | ||
82 | s64 switch_max; | ||
83 | s64 switch_min; | ||
84 | u64 switch_num; | ||
85 | s64 switch_avg; | ||
86 | s64 switch_std; | ||
87 | }; | ||
88 | #endif | ||
89 | |||
90 | struct nvgpu_clk_arb_target { | ||
91 | u16 mclk; | ||
92 | u16 gpc2clk; | ||
93 | u32 pstate; | ||
94 | }; | ||
95 | |||
96 | struct nvgpu_clk_arb { | ||
97 | spinlock_t sessions_lock; | ||
98 | spinlock_t users_lock; | ||
99 | |||
100 | struct mutex pstate_lock; | ||
101 | struct list_head users; | ||
102 | struct list_head sessions; | ||
103 | struct llist_head requests; | ||
104 | |||
105 | struct gk20a *g; | ||
106 | int status; | ||
107 | |||
108 | struct nvgpu_clk_arb_target actual_pool[2]; | ||
109 | struct nvgpu_clk_arb_target *actual; | ||
110 | |||
111 | u16 gpc2clk_default_mhz; | ||
112 | u16 mclk_default_mhz; | ||
113 | u32 voltuv_actual; | ||
114 | |||
115 | struct work_struct update_fn_work; | ||
116 | struct workqueue_struct *update_work_queue; | ||
117 | struct work_struct vf_table_fn_work; | ||
118 | struct workqueue_struct *vf_table_work_queue; | ||
119 | |||
120 | wait_queue_head_t request_wq; | ||
121 | |||
122 | struct nvgpu_clk_vf_table *current_vf_table; | ||
123 | struct nvgpu_clk_vf_table vf_table_pool[2]; | ||
124 | u32 vf_table_index; | ||
125 | |||
126 | u16 *mclk_f_points; | ||
127 | atomic_t req_nr; | ||
128 | |||
129 | u32 mclk_f_numpoints; | ||
130 | u16 *gpc2clk_f_points; | ||
131 | u32 gpc2clk_f_numpoints; | ||
132 | |||
133 | #ifdef CONFIG_DEBUG_FS | ||
134 | struct nvgpu_clk_arb_debug debug_pool[2]; | ||
135 | struct nvgpu_clk_arb_debug *debug; | ||
136 | bool debugfs_set; | ||
137 | #endif | ||
138 | }; | ||
139 | |||
140 | struct nvgpu_clk_dev { | ||
141 | struct nvgpu_clk_session *session; | ||
142 | union { | ||
143 | struct list_head link; | ||
144 | struct llist_node node; | ||
145 | }; | ||
146 | wait_queue_head_t readout_wq; | ||
147 | atomic_t poll_mask; | ||
148 | u16 gpc2clk_target_mhz; | ||
149 | u16 mclk_target_mhz; | ||
150 | struct kref refcount; | ||
151 | }; | ||
152 | |||
153 | struct nvgpu_clk_session { | ||
154 | bool zombie; | ||
155 | struct gk20a *g; | ||
156 | struct kref refcount; | ||
157 | struct list_head link; | ||
158 | struct llist_head targets; | ||
159 | |||
160 | struct nvgpu_clk_arb_target target_pool[2]; | ||
161 | struct nvgpu_clk_arb_target *target; | ||
162 | }; | ||
163 | |||
164 | static const struct file_operations completion_dev_ops = { | ||
165 | .owner = THIS_MODULE, | ||
166 | .release = nvgpu_clk_arb_release_completion_dev, | ||
167 | .poll = nvgpu_clk_arb_poll_dev, | ||
168 | }; | ||
169 | |||
170 | static const struct file_operations event_dev_ops = { | ||
171 | .owner = THIS_MODULE, | ||
172 | .release = nvgpu_clk_arb_release_event_dev, | ||
173 | .poll = nvgpu_clk_arb_poll_dev, | ||
174 | }; | ||
175 | |||
176 | int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | ||
177 | { | ||
178 | struct nvgpu_clk_arb *arb; | ||
179 | u16 default_mhz; | ||
180 | int err; | ||
181 | int index; | ||
182 | struct nvgpu_clk_vf_table *table; | ||
183 | |||
184 | gk20a_dbg_fn(""); | ||
185 | |||
186 | if (!g->ops.clk_arb.get_arbiter_clk_domains) | ||
187 | return 0; | ||
188 | |||
189 | arb = kzalloc(sizeof(struct nvgpu_clk_arb), GFP_KERNEL); | ||
190 | if (!arb) { | ||
191 | err = -ENOMEM; | ||
192 | goto init_fail; | ||
193 | } | ||
194 | |||
195 | arb->mclk_f_points = kcalloc(MAX_F_POINTS, sizeof(u16), GFP_KERNEL); | ||
196 | if (!arb->mclk_f_points) { | ||
197 | err = -ENOMEM; | ||
198 | goto init_fail; | ||
199 | } | ||
200 | |||
201 | arb->gpc2clk_f_points = kcalloc(MAX_F_POINTS, sizeof(u16), GFP_KERNEL); | ||
202 | if (!arb->gpc2clk_f_points) { | ||
203 | err = -ENOMEM; | ||
204 | goto init_fail; | ||
205 | } | ||
206 | |||
207 | for (index = 0; index < 2; index++) { | ||
208 | table = &arb->vf_table_pool[index]; | ||
209 | table->gpc2clk_num_points = MAX_F_POINTS; | ||
210 | table->mclk_num_points = MAX_F_POINTS; | ||
211 | |||
212 | table->gpc2clk_points = kcalloc(MAX_F_POINTS, | ||
213 | sizeof(struct nvgpu_clk_vf_point), GFP_KERNEL); | ||
214 | if (!table->gpc2clk_points) { | ||
215 | err = -ENOMEM; | ||
216 | goto init_fail; | ||
217 | } | ||
218 | |||
219 | |||
220 | table->mclk_points = kcalloc(MAX_F_POINTS, | ||
221 | sizeof(struct nvgpu_clk_vf_point), GFP_KERNEL); | ||
222 | if (!table->mclk_points) { | ||
223 | err = -ENOMEM; | ||
224 | goto init_fail; | ||
225 | } | ||
226 | } | ||
227 | |||
228 | g->clk_arb = arb; | ||
229 | arb->g = g; | ||
230 | |||
231 | mutex_init(&arb->pstate_lock); | ||
232 | spin_lock_init(&arb->sessions_lock); | ||
233 | spin_lock_init(&arb->users_lock); | ||
234 | |||
235 | err = g->ops.clk_arb.get_arbiter_clk_default(g, | ||
236 | NVGPU_GPU_CLK_DOMAIN_MCLK, &default_mhz); | ||
237 | if (err) { | ||
238 | err = -EINVAL; | ||
239 | goto init_fail; | ||
240 | } | ||
241 | |||
242 | arb->mclk_default_mhz = default_mhz; | ||
243 | |||
244 | err = g->ops.clk_arb.get_arbiter_clk_default(g, | ||
245 | NVGPU_GPU_CLK_DOMAIN_GPC2CLK, &default_mhz); | ||
246 | if (err) { | ||
247 | err = -EINVAL; | ||
248 | goto init_fail; | ||
249 | } | ||
250 | |||
251 | arb->gpc2clk_default_mhz = default_mhz; | ||
252 | |||
253 | arb->actual = &arb->actual_pool[0]; | ||
254 | |||
255 | atomic_set(&arb->req_nr, 0); | ||
256 | |||
257 | INIT_LIST_HEAD_RCU(&arb->users); | ||
258 | INIT_LIST_HEAD_RCU(&arb->sessions); | ||
259 | init_llist_head(&arb->requests); | ||
260 | |||
261 | init_waitqueue_head(&arb->request_wq); | ||
262 | arb->vf_table_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1, | ||
263 | "vf_table_update"); | ||
264 | arb->update_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1, | ||
265 | "arbiter_update"); | ||
266 | |||
267 | |||
268 | INIT_WORK(&arb->vf_table_fn_work, nvgpu_clk_arb_run_vf_table_cb); | ||
269 | |||
270 | INIT_WORK(&arb->update_fn_work, nvgpu_clk_arb_run_arbiter_cb); | ||
271 | |||
272 | #ifdef CONFIG_DEBUG_FS | ||
273 | arb->debug = &arb->debug_pool[0]; | ||
274 | |||
275 | if (!arb->debugfs_set) { | ||
276 | if (nvgpu_clk_arb_debugfs_init(g)) | ||
277 | arb->debugfs_set = true; | ||
278 | } | ||
279 | #endif | ||
280 | err = clk_vf_point_cache(g); | ||
281 | if (err < 0) | ||
282 | goto init_fail; | ||
283 | |||
284 | err = nvgpu_clk_arb_update_vf_table(arb); | ||
285 | if (err < 0) | ||
286 | goto init_fail; | ||
287 | do { | ||
288 | /* Check that first run is completed */ | ||
289 | smp_mb(); | ||
290 | wait_event_interruptible(arb->request_wq, | ||
291 | atomic_read(&arb->req_nr)); | ||
292 | } while (!atomic_read(&arb->req_nr)); | ||
293 | |||
294 | |||
295 | return arb->status; | ||
296 | |||
297 | init_fail: | ||
298 | |||
299 | kfree(arb->gpc2clk_f_points); | ||
300 | kfree(arb->mclk_f_points); | ||
301 | |||
302 | for (index = 0; index < 2; index++) { | ||
303 | kfree(arb->vf_table_pool[index].gpc2clk_points); | ||
304 | kfree(arb->vf_table_pool[index].mclk_points); | ||
305 | } | ||
306 | |||
307 | kfree(arb); | ||
308 | |||
309 | return err; | ||
310 | } | ||
311 | |||
312 | void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g) | ||
313 | { | ||
314 | kfree(g->clk_arb); | ||
315 | } | ||
316 | |||
317 | static int nvgpu_clk_arb_install_fd(struct gk20a *g, | ||
318 | struct nvgpu_clk_session *session, | ||
319 | const struct file_operations *fops, | ||
320 | struct nvgpu_clk_dev **_dev) | ||
321 | { | ||
322 | struct file *file; | ||
323 | char *name; | ||
324 | int fd; | ||
325 | int err; | ||
326 | struct nvgpu_clk_dev *dev; | ||
327 | |||
328 | gk20a_dbg_fn(""); | ||
329 | |||
330 | dev = kzalloc(sizeof(*dev), GFP_KERNEL); | ||
331 | if (!dev) | ||
332 | return -ENOMEM; | ||
333 | |||
334 | fd = get_unused_fd_flags(O_RDWR); | ||
335 | if (fd < 0) | ||
336 | return fd; | ||
337 | |||
338 | name = kasprintf(GFP_KERNEL, "%s-clk-fd%d", dev_name(g->dev), fd); | ||
339 | file = anon_inode_getfile(name, fops, dev, O_RDWR); | ||
340 | kfree(name); | ||
341 | if (IS_ERR(file)) { | ||
342 | err = PTR_ERR(file); | ||
343 | goto fail; | ||
344 | } | ||
345 | |||
346 | fd_install(fd, file); | ||
347 | |||
348 | init_waitqueue_head(&dev->readout_wq); | ||
349 | atomic_set(&dev->poll_mask, 0); | ||
350 | |||
351 | dev->session = session; | ||
352 | kref_init(&dev->refcount); | ||
353 | |||
354 | kref_get(&session->refcount); | ||
355 | |||
356 | *_dev = dev; | ||
357 | |||
358 | return fd; | ||
359 | |||
360 | fail: | ||
361 | kfree(dev); | ||
362 | put_unused_fd(fd); | ||
363 | |||
364 | return err; | ||
365 | } | ||
366 | |||
367 | int nvgpu_clk_arb_init_session(struct gk20a *g, | ||
368 | struct nvgpu_clk_session **_session) | ||
369 | { | ||
370 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
371 | struct nvgpu_clk_session *session = *(_session); | ||
372 | |||
373 | gk20a_dbg_fn(""); | ||
374 | |||
375 | if (!g->ops.clk_arb.get_arbiter_clk_domains) | ||
376 | return 0; | ||
377 | |||
378 | session = kzalloc(sizeof(struct nvgpu_clk_session), GFP_KERNEL); | ||
379 | if (!session) | ||
380 | return -ENOMEM; | ||
381 | session->g = g; | ||
382 | |||
383 | kref_init(&session->refcount); | ||
384 | |||
385 | session->zombie = false; | ||
386 | session->target_pool[0].pstate = CTRL_PERF_PSTATE_P8; | ||
387 | /* make sure that the initialization of the pool is visible | ||
388 | * before the update */ | ||
389 | smp_wmb(); | ||
390 | session->target = &session->target_pool[0]; | ||
391 | |||
392 | init_llist_head(&session->targets); | ||
393 | |||
394 | spin_lock(&arb->sessions_lock); | ||
395 | list_add_tail_rcu(&session->link, &arb->sessions); | ||
396 | spin_unlock(&arb->sessions_lock); | ||
397 | |||
398 | *_session = session; | ||
399 | |||
400 | return 0; | ||
401 | } | ||
402 | |||
403 | static void nvgpu_clk_arb_free_fd(struct kref *refcount) | ||
404 | { | ||
405 | struct nvgpu_clk_dev *dev = container_of(refcount, | ||
406 | struct nvgpu_clk_dev, refcount); | ||
407 | |||
408 | kfree(dev); | ||
409 | } | ||
410 | |||
411 | static void nvgpu_clk_arb_free_session(struct kref *refcount) | ||
412 | { | ||
413 | struct nvgpu_clk_session *session = container_of(refcount, | ||
414 | struct nvgpu_clk_session, refcount); | ||
415 | struct nvgpu_clk_arb *arb = session->g->clk_arb; | ||
416 | struct nvgpu_clk_dev *dev, *tmp; | ||
417 | struct llist_node *head; | ||
418 | |||
419 | gk20a_dbg_fn(""); | ||
420 | |||
421 | spin_lock(&arb->sessions_lock); | ||
422 | list_del_rcu(&session->link); | ||
423 | spin_unlock(&arb->sessions_lock); | ||
424 | |||
425 | head = llist_del_all(&session->targets); | ||
426 | llist_for_each_entry_safe(dev, tmp, head, node) { | ||
427 | kref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | ||
428 | } | ||
429 | synchronize_rcu(); | ||
430 | kfree(session); | ||
431 | } | ||
432 | |||
433 | void nvgpu_clk_arb_release_session(struct gk20a *g, | ||
434 | struct nvgpu_clk_session *session) | ||
435 | { | ||
436 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
437 | |||
438 | gk20a_dbg_fn(""); | ||
439 | |||
440 | session->zombie = true; | ||
441 | kref_put(&session->refcount, nvgpu_clk_arb_free_session); | ||
442 | |||
443 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
444 | } | ||
445 | |||
446 | int nvgpu_clk_arb_install_event_fd(struct gk20a *g, | ||
447 | struct nvgpu_clk_session *session, int *event_fd) | ||
448 | { | ||
449 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
450 | struct nvgpu_clk_dev *dev; | ||
451 | int fd; | ||
452 | |||
453 | gk20a_dbg_fn(""); | ||
454 | |||
455 | fd = nvgpu_clk_arb_install_fd(g, session, &event_dev_ops, &dev); | ||
456 | if (fd < 0) | ||
457 | return fd; | ||
458 | |||
459 | spin_lock(&arb->users_lock); | ||
460 | list_add_tail_rcu(&dev->link, &arb->users); | ||
461 | spin_unlock(&arb->users_lock); | ||
462 | |||
463 | *event_fd = fd; | ||
464 | |||
465 | return 0; | ||
466 | } | ||
467 | |||
468 | int nvgpu_clk_arb_install_request_fd(struct gk20a *g, | ||
469 | struct nvgpu_clk_session *session, int *request_fd) | ||
470 | { | ||
471 | struct nvgpu_clk_dev *dev; | ||
472 | int fd; | ||
473 | |||
474 | gk20a_dbg_fn(""); | ||
475 | |||
476 | fd = nvgpu_clk_arb_install_fd(g, session, &completion_dev_ops, &dev); | ||
477 | if (fd < 0) | ||
478 | return fd; | ||
479 | |||
480 | *request_fd = fd; | ||
481 | |||
482 | return 0; | ||
483 | } | ||
484 | |||
485 | static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb) | ||
486 | { | ||
487 | struct gk20a *g = arb->g; | ||
488 | struct nvgpu_clk_vf_table *table; | ||
489 | |||
490 | u32 i, j; | ||
491 | int status = -EINVAL; | ||
492 | u32 gpc2clk_voltuv = 0, mclk_voltuv = 0; | ||
493 | u32 gpc2clk_voltuv_sram = 0, mclk_voltuv_sram = 0; | ||
494 | u16 gpc2clk_min, gpc2clk_max, clk_cur; | ||
495 | u16 mclk_min, mclk_max; | ||
496 | u32 num_points; | ||
497 | |||
498 | struct clk_set_info *p5_info, *p0_info; | ||
499 | |||
500 | |||
501 | table = ACCESS_ONCE(arb->current_vf_table); | ||
502 | /* make flag visible when all data has resolved in the tables */ | ||
503 | smp_rmb(); | ||
504 | |||
505 | table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] : | ||
506 | &arb->vf_table_pool[0]; | ||
507 | |||
508 | /* Get allowed memory ranges */ | ||
509 | if (nvgpu_clk_arb_get_arbiter_clk_range(g, NVGPU_GPU_CLK_DOMAIN_GPC2CLK, | ||
510 | &gpc2clk_min, | ||
511 | &gpc2clk_max) < 0) { | ||
512 | gk20a_err(dev_from_gk20a(g), | ||
513 | "failed to fetch GPC2CLK range"); | ||
514 | goto exit_vf_table; | ||
515 | } | ||
516 | if (nvgpu_clk_arb_get_arbiter_clk_range(g, NVGPU_GPU_CLK_DOMAIN_MCLK, | ||
517 | &mclk_min, &mclk_max) < 0) { | ||
518 | gk20a_err(dev_from_gk20a(g), | ||
519 | "failed to fetch MCLK range"); | ||
520 | goto exit_vf_table; | ||
521 | } | ||
522 | |||
523 | if (clk_domain_get_f_points(arb->g, NVGPU_GPU_CLK_DOMAIN_GPC2CLK, | ||
524 | &table->gpc2clk_num_points, arb->gpc2clk_f_points)) { | ||
525 | gk20a_err(dev_from_gk20a(g), | ||
526 | "failed to fetch GPC2CLK frequency points"); | ||
527 | goto exit_vf_table; | ||
528 | } | ||
529 | |||
530 | table->gpc2clk_num_points = MAX_F_POINTS; | ||
531 | table->mclk_num_points = MAX_F_POINTS; | ||
532 | |||
533 | if (clk_domain_get_f_points(arb->g, NVGPU_GPU_CLK_DOMAIN_MCLK, | ||
534 | &table->mclk_num_points, arb->mclk_f_points)) { | ||
535 | gk20a_err(dev_from_gk20a(g), | ||
536 | "failed to fetch MCLK frequency points"); | ||
537 | goto exit_vf_table; | ||
538 | } | ||
539 | if (!table->mclk_num_points || !table->gpc2clk_num_points) { | ||
540 | gk20a_err(dev_from_gk20a(g), | ||
541 | "empty queries to f points mclk %d gpc2clk %d", | ||
542 | table->mclk_num_points, table->gpc2clk_num_points); | ||
543 | status = -EINVAL; | ||
544 | goto exit_vf_table; | ||
545 | } | ||
546 | |||
547 | memset(table->mclk_points, 0, | ||
548 | table->mclk_num_points*sizeof(struct nvgpu_clk_vf_point)); | ||
549 | memset(table->gpc2clk_points, 0, | ||
550 | table->gpc2clk_num_points*sizeof(struct nvgpu_clk_vf_point)); | ||
551 | |||
552 | p5_info = pstate_get_clk_set_info(g, | ||
553 | CTRL_PERF_PSTATE_P5, clkwhich_mclk); | ||
554 | if (!p5_info) { | ||
555 | gk20a_err(dev_from_gk20a(g), | ||
556 | "failed to get MCLK P5 info"); | ||
557 | goto exit_vf_table; | ||
558 | } | ||
559 | p0_info = pstate_get_clk_set_info(g, | ||
560 | CTRL_PERF_PSTATE_P0, clkwhich_mclk); | ||
561 | if (!p0_info) { | ||
562 | gk20a_err(dev_from_gk20a(g), | ||
563 | "failed to get MCLK P0 info"); | ||
564 | goto exit_vf_table; | ||
565 | } | ||
566 | |||
567 | for (i = 0, j = 0, num_points = 0, clk_cur = 0; | ||
568 | i < table->mclk_num_points; i++) { | ||
569 | |||
570 | if ((arb->mclk_f_points[i] >= mclk_min) && | ||
571 | (arb->mclk_f_points[i] <= mclk_max) && | ||
572 | (arb->mclk_f_points[i] != clk_cur)) { | ||
573 | |||
574 | table->mclk_points[j].mem_mhz = arb->mclk_f_points[i]; | ||
575 | mclk_voltuv = mclk_voltuv_sram = 0; | ||
576 | |||
577 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK, | ||
578 | &table->mclk_points[j].mem_mhz, &mclk_voltuv, | ||
579 | CTRL_VOLT_DOMAIN_LOGIC); | ||
580 | if (status < 0) { | ||
581 | gk20a_err(dev_from_gk20a(g), | ||
582 | "failed to get MCLK LOGIC voltage"); | ||
583 | goto exit_vf_table; | ||
584 | } | ||
585 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK, | ||
586 | &table->mclk_points[j].mem_mhz, | ||
587 | &mclk_voltuv_sram, | ||
588 | CTRL_VOLT_DOMAIN_SRAM); | ||
589 | if (status < 0) { | ||
590 | gk20a_err(dev_from_gk20a(g), | ||
591 | "failed to get MCLK SRAM voltage"); | ||
592 | goto exit_vf_table; | ||
593 | } | ||
594 | |||
595 | table->mclk_points[j].uvolt = mclk_voltuv; | ||
596 | table->mclk_points[j].uvolt_sram = mclk_voltuv_sram; | ||
597 | clk_cur = table->mclk_points[j].mem_mhz; | ||
598 | |||
599 | if ((clk_cur >= p5_info->min_mhz) && | ||
600 | (clk_cur <= p5_info->max_mhz)) | ||
601 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
602 | &table->mclk_points[j], | ||
603 | CTRL_PERF_PSTATE_P5); | ||
604 | if ((clk_cur >= p0_info->min_mhz) && | ||
605 | (clk_cur <= p0_info->max_mhz)) | ||
606 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
607 | &table->mclk_points[j], | ||
608 | CTRL_PERF_PSTATE_P0); | ||
609 | |||
610 | j++; | ||
611 | num_points++; | ||
612 | |||
613 | } | ||
614 | } | ||
615 | table->mclk_num_points = num_points; | ||
616 | |||
617 | p5_info = pstate_get_clk_set_info(g, | ||
618 | CTRL_PERF_PSTATE_P5, clkwhich_gpc2clk); | ||
619 | if (!p5_info) { | ||
620 | status = -EINVAL; | ||
621 | gk20a_err(dev_from_gk20a(g), | ||
622 | "failed to get GPC2CLK P5 info"); | ||
623 | goto exit_vf_table; | ||
624 | } | ||
625 | |||
626 | p0_info = pstate_get_clk_set_info(g, | ||
627 | CTRL_PERF_PSTATE_P0, clkwhich_gpc2clk); | ||
628 | if (!p0_info) { | ||
629 | status = -EINVAL; | ||
630 | gk20a_err(dev_from_gk20a(g), | ||
631 | "failed to get GPC2CLK P0 info"); | ||
632 | goto exit_vf_table; | ||
633 | } | ||
634 | |||
635 | /* GPC2CLK needs to be checked in two passes. The first determines the | ||
636 | * relationships between GPC2CLK, SYS2CLK and XBAR2CLK, while the | ||
637 | * second verifies that the clocks minimum is satisfied and sets | ||
638 | * the voltages | ||
639 | */ | ||
640 | for (i = 0, j = 0, num_points = 0, clk_cur = 0; | ||
641 | i < table->gpc2clk_num_points; i++) { | ||
642 | struct set_fll_clk setfllclk; | ||
643 | |||
644 | if ((arb->gpc2clk_f_points[i] >= gpc2clk_min) && | ||
645 | (arb->gpc2clk_f_points[i] <= gpc2clk_max) && | ||
646 | (arb->gpc2clk_f_points[i] != clk_cur)) { | ||
647 | |||
648 | table->gpc2clk_points[j].gpc_mhz = | ||
649 | arb->gpc2clk_f_points[i]; | ||
650 | setfllclk.gpc2clkmhz = arb->gpc2clk_f_points[i]; | ||
651 | status = clk_get_fll_clks(g, &setfllclk); | ||
652 | if (status < 0) { | ||
653 | gk20a_err(dev_from_gk20a(g), | ||
654 | "failed to get GPC2CLK slave clocks"); | ||
655 | goto exit_vf_table; | ||
656 | } | ||
657 | |||
658 | table->gpc2clk_points[j].sys_mhz = | ||
659 | setfllclk.sys2clkmhz; | ||
660 | table->gpc2clk_points[j].xbar_mhz = | ||
661 | setfllclk.xbar2clkmhz; | ||
662 | |||
663 | clk_cur = table->gpc2clk_points[j].gpc_mhz; | ||
664 | |||
665 | if ((clk_cur >= p5_info->min_mhz) && | ||
666 | (clk_cur <= p5_info->max_mhz)) | ||
667 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
668 | &table->gpc2clk_points[j], | ||
669 | CTRL_PERF_PSTATE_P5); | ||
670 | if ((clk_cur >= p0_info->min_mhz) && | ||
671 | (clk_cur <= p0_info->max_mhz)) | ||
672 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
673 | &table->gpc2clk_points[j], | ||
674 | CTRL_PERF_PSTATE_P0); | ||
675 | |||
676 | j++; | ||
677 | num_points++; | ||
678 | } | ||
679 | } | ||
680 | table->gpc2clk_num_points = num_points; | ||
681 | |||
682 | /* Second pass */ | ||
683 | for (i = 0, j = 0; i < table->gpc2clk_num_points; i++) { | ||
684 | |||
685 | u16 alt_gpc2clk = table->gpc2clk_points[i].gpc_mhz; | ||
686 | gpc2clk_voltuv = gpc2clk_voltuv_sram = 0; | ||
687 | |||
688 | /* Check sysclk */ | ||
689 | p5_info = pstate_get_clk_set_info(g, | ||
690 | VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]), | ||
691 | clkwhich_sys2clk); | ||
692 | if (!p5_info) { | ||
693 | status = -EINVAL; | ||
694 | gk20a_err(dev_from_gk20a(g), | ||
695 | "failed to get SYS2CLK P5 info"); | ||
696 | goto exit_vf_table; | ||
697 | } | ||
698 | |||
699 | /* sys2clk below clk min, need to find correct clock */ | ||
700 | if (table->gpc2clk_points[i].sys_mhz < p5_info->min_mhz) { | ||
701 | for (j = i + 1; j < table->gpc2clk_num_points; j++) { | ||
702 | |||
703 | if (table->gpc2clk_points[j].sys_mhz >= | ||
704 | p5_info->min_mhz) { | ||
705 | |||
706 | |||
707 | table->gpc2clk_points[i].sys_mhz = | ||
708 | p5_info->min_mhz; | ||
709 | |||
710 | alt_gpc2clk = alt_gpc2clk < | ||
711 | table->gpc2clk_points[j]. | ||
712 | gpc_mhz ? | ||
713 | table->gpc2clk_points[j]. | ||
714 | gpc_mhz: | ||
715 | alt_gpc2clk; | ||
716 | break; | ||
717 | } | ||
718 | } | ||
719 | /* no VF exists that satisfies condition */ | ||
720 | if (j == table->gpc2clk_num_points) { | ||
721 | gk20a_err(dev_from_gk20a(g), | ||
722 | "NO SYS2CLK VF point possible"); | ||
723 | status = -EINVAL; | ||
724 | goto exit_vf_table; | ||
725 | } | ||
726 | } | ||
727 | |||
728 | /* Check xbarclk */ | ||
729 | p5_info = pstate_get_clk_set_info(g, | ||
730 | VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]), | ||
731 | clkwhich_xbar2clk); | ||
732 | if (!p5_info) { | ||
733 | status = -EINVAL; | ||
734 | gk20a_err(dev_from_gk20a(g), | ||
735 | "failed to get SYS2CLK P5 info"); | ||
736 | goto exit_vf_table; | ||
737 | } | ||
738 | |||
739 | /* xbar2clk below clk min, need to find correct clock */ | ||
740 | if (table->gpc2clk_points[i].xbar_mhz < p5_info->min_mhz) { | ||
741 | for (j = i; j < table->gpc2clk_num_points; j++) { | ||
742 | if (table->gpc2clk_points[j].xbar_mhz >= | ||
743 | p5_info->min_mhz) { | ||
744 | |||
745 | table->gpc2clk_points[i].xbar_mhz = | ||
746 | p5_info->min_mhz; | ||
747 | |||
748 | alt_gpc2clk = alt_gpc2clk < | ||
749 | table->gpc2clk_points[j]. | ||
750 | gpc_mhz ? | ||
751 | table->gpc2clk_points[j]. | ||
752 | gpc_mhz: | ||
753 | alt_gpc2clk; | ||
754 | break; | ||
755 | } | ||
756 | } | ||
757 | /* no VF exists that satisfies condition */ | ||
758 | if (j == table->gpc2clk_num_points) { | ||
759 | status = -EINVAL; | ||
760 | gk20a_err(dev_from_gk20a(g), | ||
761 | "NO XBAR2CLK VF point possible"); | ||
762 | |||
763 | goto exit_vf_table; | ||
764 | } | ||
765 | } | ||
766 | |||
767 | /* Calculate voltages */ | ||
768 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
769 | &alt_gpc2clk, &gpc2clk_voltuv, | ||
770 | CTRL_VOLT_DOMAIN_LOGIC); | ||
771 | if (status < 0) { | ||
772 | gk20a_err(dev_from_gk20a(g), | ||
773 | "failed to get GPC2CLK LOGIC voltage"); | ||
774 | goto exit_vf_table; | ||
775 | } | ||
776 | |||
777 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
778 | &alt_gpc2clk, | ||
779 | &gpc2clk_voltuv_sram, | ||
780 | CTRL_VOLT_DOMAIN_SRAM); | ||
781 | if (status < 0) { | ||
782 | gk20a_err(dev_from_gk20a(g), | ||
783 | "failed to get GPC2CLK SRAM voltage"); | ||
784 | goto exit_vf_table; | ||
785 | } | ||
786 | |||
787 | table->gpc2clk_points[i].uvolt = gpc2clk_voltuv; | ||
788 | table->gpc2clk_points[i].uvolt_sram = gpc2clk_voltuv_sram; | ||
789 | } | ||
790 | |||
791 | /* make table visible when all data has resolved in the tables */ | ||
792 | smp_wmb(); | ||
793 | xchg(&arb->current_vf_table, table); | ||
794 | |||
795 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
796 | exit_vf_table: | ||
797 | |||
798 | return status; | ||
799 | } | ||
800 | |||
801 | void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g) | ||
802 | { | ||
803 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
804 | |||
805 | queue_work(arb->vf_table_work_queue, &arb->vf_table_fn_work); | ||
806 | } | ||
807 | |||
808 | static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work) | ||
809 | { | ||
810 | struct nvgpu_clk_arb *arb = | ||
811 | container_of(work, struct nvgpu_clk_arb, vf_table_fn_work); | ||
812 | struct gk20a *g = arb->g; | ||
813 | u32 err; | ||
814 | |||
815 | /* get latest vf curve from pmu */ | ||
816 | err = clk_vf_point_cache(g); | ||
817 | if (err) { | ||
818 | gk20a_err(dev_from_gk20a(g), | ||
819 | "failed to cache VF table"); | ||
820 | return; | ||
821 | } | ||
822 | nvgpu_clk_arb_update_vf_table(arb); | ||
823 | } | ||
824 | |||
825 | static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | ||
826 | { | ||
827 | struct nvgpu_clk_arb *arb = | ||
828 | container_of(work, struct nvgpu_clk_arb, update_fn_work); | ||
829 | struct nvgpu_clk_session *session; | ||
830 | struct nvgpu_clk_dev *dev; | ||
831 | struct nvgpu_clk_dev *tmp; | ||
832 | struct nvgpu_clk_arb_target *target, *actual; | ||
833 | struct gk20a *g = arb->g; | ||
834 | struct llist_node *head; | ||
835 | |||
836 | u32 pstate = VF_POINT_INVALID_PSTATE; | ||
837 | u32 voltuv, voltuv_sram; | ||
838 | bool mclk_set, gpc2clk_set; | ||
839 | u32 nuvmin, nuvmin_sram; | ||
840 | |||
841 | int status = 0; | ||
842 | |||
843 | /* Temporary variables for checking target frequency */ | ||
844 | u16 gpc2clk_target, sys2clk_target, xbar2clk_target, mclk_target; | ||
845 | |||
846 | #ifdef CONFIG_DEBUG_FS | ||
847 | u64 t0, t1; | ||
848 | struct nvgpu_clk_arb_debug *debug; | ||
849 | |||
850 | #endif | ||
851 | |||
852 | gk20a_dbg_fn(""); | ||
853 | |||
854 | #ifdef CONFIG_DEBUG_FS | ||
855 | g->ops.read_ptimer(g, &t0); | ||
856 | #endif | ||
857 | |||
858 | /* Only one arbiter should be running */ | ||
859 | gpc2clk_target = 0; | ||
860 | mclk_target = 0; | ||
861 | |||
862 | rcu_read_lock(); | ||
863 | list_for_each_entry_rcu(session, &arb->sessions, link) { | ||
864 | if (!session->zombie) { | ||
865 | mclk_set = false; | ||
866 | gpc2clk_set = false; | ||
867 | target = ACCESS_ONCE(session->target) == | ||
868 | &session->target_pool[0] ? | ||
869 | &session->target_pool[1] : | ||
870 | &session->target_pool[0]; | ||
871 | /* Do not reorder pointer */ | ||
872 | smp_rmb(); | ||
873 | head = llist_del_all(&session->targets); | ||
874 | if (head) { | ||
875 | |||
876 | /* Copy over state */ | ||
877 | target->mclk = session->target->mclk; | ||
878 | target->gpc2clk = session->target->gpc2clk; | ||
879 | /* Query the latest committed request */ | ||
880 | llist_for_each_entry_safe(dev, tmp, head, | ||
881 | node) { | ||
882 | if (!mclk_set && dev->mclk_target_mhz) { | ||
883 | target->mclk = | ||
884 | dev->mclk_target_mhz; | ||
885 | mclk_set = true; | ||
886 | } | ||
887 | if (!gpc2clk_set && | ||
888 | dev->gpc2clk_target_mhz) { | ||
889 | target->gpc2clk = | ||
890 | dev->gpc2clk_target_mhz; | ||
891 | gpc2clk_set = true; | ||
892 | } | ||
893 | kref_get(&dev->refcount); | ||
894 | llist_add(&dev->node, &arb->requests); | ||
895 | } | ||
896 | /* Ensure target is updated before ptr sawp */ | ||
897 | smp_wmb(); | ||
898 | xchg(&session->target, target); | ||
899 | } | ||
900 | |||
901 | mclk_target = mclk_target > session->target->mclk ? | ||
902 | mclk_target : session->target->mclk; | ||
903 | |||
904 | gpc2clk_target = | ||
905 | gpc2clk_target > session->target->gpc2clk ? | ||
906 | gpc2clk_target : session->target->gpc2clk; | ||
907 | } | ||
908 | } | ||
909 | rcu_read_unlock(); | ||
910 | |||
911 | gpc2clk_target = (gpc2clk_target > 0) ? gpc2clk_target : | ||
912 | arb->gpc2clk_default_mhz; | ||
913 | |||
914 | mclk_target = (mclk_target > 0) ? mclk_target: | ||
915 | arb->mclk_default_mhz; | ||
916 | |||
917 | sys2clk_target = 0; | ||
918 | xbar2clk_target = 0; | ||
919 | /* Query the table for the closest vf point to program */ | ||
920 | pstate = nvgpu_clk_arb_find_vf_point(arb, &gpc2clk_target, | ||
921 | &sys2clk_target, &xbar2clk_target, &mclk_target, &voltuv, | ||
922 | &voltuv_sram, &nuvmin, &nuvmin_sram); | ||
923 | |||
924 | if (pstate == VF_POINT_INVALID_PSTATE) { | ||
925 | arb->status = -EINVAL; | ||
926 | /* make status visible */ | ||
927 | smp_mb(); | ||
928 | goto exit_arb; | ||
929 | } | ||
930 | |||
931 | if ((arb->actual->gpc2clk == gpc2clk_target) && | ||
932 | (arb->actual->mclk == mclk_target) && | ||
933 | (arb->voltuv_actual == voltuv)) { | ||
934 | goto exit_arb; | ||
935 | } | ||
936 | |||
937 | /* Program clocks */ | ||
938 | /* A change in both mclk of gpc2clk may require a change in voltage */ | ||
939 | |||
940 | mutex_lock(&arb->pstate_lock); | ||
941 | status = nvgpu_lpwr_disable_pg(g, false); | ||
942 | |||
943 | status = clk_pmu_freq_controller_load(g, false); | ||
944 | if (status < 0) { | ||
945 | arb->status = status; | ||
946 | mutex_unlock(&arb->pstate_lock); | ||
947 | |||
948 | /* make status visible */ | ||
949 | smp_mb(); | ||
950 | goto exit_arb; | ||
951 | } | ||
952 | status = volt_set_noiseaware_vmin(g, nuvmin, nuvmin_sram); | ||
953 | if (status < 0) { | ||
954 | arb->status = status; | ||
955 | mutex_unlock(&arb->pstate_lock); | ||
956 | |||
957 | /* make status visible */ | ||
958 | smp_mb(); | ||
959 | goto exit_arb; | ||
960 | } | ||
961 | |||
962 | status = nvgpu_clk_arb_change_vf_point(g, gpc2clk_target, | ||
963 | sys2clk_target, xbar2clk_target, mclk_target, voltuv, | ||
964 | voltuv_sram); | ||
965 | if (status < 0) { | ||
966 | arb->status = status; | ||
967 | mutex_unlock(&arb->pstate_lock); | ||
968 | |||
969 | /* make status visible */ | ||
970 | smp_mb(); | ||
971 | goto exit_arb; | ||
972 | } | ||
973 | |||
974 | status = clk_pmu_freq_controller_load(g, true); | ||
975 | if (status < 0) { | ||
976 | arb->status = status; | ||
977 | mutex_unlock(&arb->pstate_lock); | ||
978 | |||
979 | /* make status visible */ | ||
980 | smp_mb(); | ||
981 | goto exit_arb; | ||
982 | } | ||
983 | |||
984 | status = nvgpu_lwpr_mclk_change(g, pstate); | ||
985 | if (status < 0) { | ||
986 | arb->status = status; | ||
987 | mutex_unlock(&arb->pstate_lock); | ||
988 | |||
989 | /* make status visible */ | ||
990 | smp_mb(); | ||
991 | goto exit_arb; | ||
992 | } | ||
993 | |||
994 | actual = ACCESS_ONCE(arb->actual) == &arb->actual_pool[0] ? | ||
995 | &arb->actual_pool[1] : &arb->actual_pool[0]; | ||
996 | |||
997 | /* do not reorder this pointer */ | ||
998 | smp_rmb(); | ||
999 | actual->gpc2clk = gpc2clk_target; | ||
1000 | actual->mclk = mclk_target; | ||
1001 | arb->voltuv_actual = voltuv; | ||
1002 | actual->pstate = pstate; | ||
1003 | arb->status = status; | ||
1004 | |||
1005 | /* Make changes visible to other threads */ | ||
1006 | smp_wmb(); | ||
1007 | xchg(&arb->actual, actual); | ||
1008 | |||
1009 | status = nvgpu_lpwr_enable_pg(g, false); | ||
1010 | if (status < 0) { | ||
1011 | arb->status = status; | ||
1012 | mutex_unlock(&arb->pstate_lock); | ||
1013 | |||
1014 | /* make status visible */ | ||
1015 | smp_mb(); | ||
1016 | goto exit_arb; | ||
1017 | } | ||
1018 | |||
1019 | /* status must be visible before atomic inc */ | ||
1020 | smp_wmb(); | ||
1021 | atomic_inc(&arb->req_nr); | ||
1022 | |||
1023 | /* Unlock pstate change for PG */ | ||
1024 | mutex_unlock(&arb->pstate_lock); | ||
1025 | |||
1026 | wake_up_interruptible(&arb->request_wq); | ||
1027 | |||
1028 | #ifdef CONFIG_DEBUG_FS | ||
1029 | g->ops.read_ptimer(g, &t1); | ||
1030 | |||
1031 | debug = arb->debug == &arb->debug_pool[0] ? | ||
1032 | &arb->debug_pool[1] : &arb->debug_pool[0]; | ||
1033 | |||
1034 | memcpy(debug, arb->debug, sizeof(arb->debug_pool[0])); | ||
1035 | debug->switch_num++; | ||
1036 | |||
1037 | if (debug->switch_num == 1) { | ||
1038 | debug->switch_max = debug->switch_min = | ||
1039 | debug->switch_avg = (t1-t0)/1000; | ||
1040 | debug->switch_std = 0; | ||
1041 | } else { | ||
1042 | s64 prev_avg; | ||
1043 | s64 curr = (t1-t0)/1000; | ||
1044 | |||
1045 | debug->switch_max = curr > debug->switch_max ? | ||
1046 | curr : debug->switch_max; | ||
1047 | debug->switch_min = debug->switch_min ? | ||
1048 | (curr < debug->switch_min ? | ||
1049 | curr : debug->switch_min) : curr; | ||
1050 | prev_avg = debug->switch_avg; | ||
1051 | debug->switch_avg = (curr + | ||
1052 | (debug->switch_avg * (debug->switch_num-1))) / | ||
1053 | debug->switch_num; | ||
1054 | debug->switch_std += | ||
1055 | (curr - debug->switch_avg) * (curr - prev_avg); | ||
1056 | } | ||
1057 | /* commit changes before exchanging debug pointer */ | ||
1058 | smp_wmb(); | ||
1059 | xchg(&arb->debug, debug); | ||
1060 | #endif | ||
1061 | |||
1062 | exit_arb: | ||
1063 | if (status < 0) | ||
1064 | gk20a_err(dev_from_gk20a(g), | ||
1065 | "Error in arbiter update"); | ||
1066 | |||
1067 | /* notify completion for all requests */ | ||
1068 | head = llist_del_all(&arb->requests); | ||
1069 | llist_for_each_entry_safe(dev, tmp, head, node) { | ||
1070 | atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM); | ||
1071 | wake_up_interruptible(&dev->readout_wq); | ||
1072 | kref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | ||
1073 | } | ||
1074 | |||
1075 | /* notify event for all users */ | ||
1076 | rcu_read_lock(); | ||
1077 | list_for_each_entry_rcu(dev, &arb->users, link) { | ||
1078 | atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM); | ||
1079 | wake_up_interruptible(&dev->readout_wq); | ||
1080 | } | ||
1081 | rcu_read_unlock(); | ||
1082 | } | ||
1083 | |||
1084 | int nvgpu_clk_arb_commit_request_fd(struct gk20a *g, | ||
1085 | struct nvgpu_clk_session *session, int request_fd) | ||
1086 | { | ||
1087 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1088 | struct nvgpu_clk_dev *dev; | ||
1089 | struct fd fd; | ||
1090 | int err = 0; | ||
1091 | |||
1092 | gk20a_dbg_fn(""); | ||
1093 | |||
1094 | fd = fdget(request_fd); | ||
1095 | |||
1096 | if (!fd.file) | ||
1097 | return -EINVAL; | ||
1098 | |||
1099 | dev = (struct nvgpu_clk_dev *) fd.file->private_data; | ||
1100 | |||
1101 | if (!dev || dev->session != session) { | ||
1102 | err = -EINVAL; | ||
1103 | goto fdput_fd; | ||
1104 | } | ||
1105 | kref_get(&dev->refcount); | ||
1106 | llist_add(&dev->node, &session->targets); | ||
1107 | |||
1108 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
1109 | |||
1110 | fdput_fd: | ||
1111 | fdput(fd); | ||
1112 | return err; | ||
1113 | } | ||
1114 | |||
1115 | static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait) | ||
1116 | { | ||
1117 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1118 | |||
1119 | gk20a_dbg_fn(""); | ||
1120 | |||
1121 | poll_wait(filp, &dev->readout_wq, wait); | ||
1122 | return atomic_xchg(&dev->poll_mask, 0); | ||
1123 | } | ||
1124 | |||
1125 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, | ||
1126 | struct file *filp) | ||
1127 | { | ||
1128 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1129 | struct nvgpu_clk_session *session = dev->session; | ||
1130 | struct nvgpu_clk_arb *arb; | ||
1131 | |||
1132 | arb = session->g->clk_arb; | ||
1133 | |||
1134 | gk20a_dbg_fn(""); | ||
1135 | |||
1136 | kref_put(&session->refcount, nvgpu_clk_arb_free_session); | ||
1137 | kref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | ||
1138 | |||
1139 | return 0; | ||
1140 | } | ||
1141 | |||
1142 | static int nvgpu_clk_arb_release_event_dev(struct inode *inode, | ||
1143 | struct file *filp) | ||
1144 | { | ||
1145 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1146 | struct nvgpu_clk_session *session = dev->session; | ||
1147 | struct nvgpu_clk_arb *arb; | ||
1148 | |||
1149 | arb = session->g->clk_arb; | ||
1150 | |||
1151 | gk20a_dbg_fn(""); | ||
1152 | |||
1153 | spin_lock(&arb->users_lock); | ||
1154 | list_del_rcu(&dev->link); | ||
1155 | spin_unlock(&arb->users_lock); | ||
1156 | |||
1157 | kref_put(&session->refcount, nvgpu_clk_arb_free_session); | ||
1158 | synchronize_rcu(); | ||
1159 | kfree(dev); | ||
1160 | |||
1161 | return 0; | ||
1162 | } | ||
1163 | |||
1164 | int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session, | ||
1165 | int request_fd, u32 api_domain, u16 target_mhz) | ||
1166 | { | ||
1167 | struct nvgpu_clk_dev *dev; | ||
1168 | struct fd fd; | ||
1169 | int err = 0; | ||
1170 | |||
1171 | gk20a_dbg_fn("domain=0x%08x target_mhz=%u", api_domain, target_mhz); | ||
1172 | |||
1173 | fd = fdget(request_fd); | ||
1174 | |||
1175 | if (!fd.file) | ||
1176 | return -EINVAL; | ||
1177 | |||
1178 | dev = fd.file->private_data; | ||
1179 | if (!dev || dev->session != session) { | ||
1180 | err = -EINVAL; | ||
1181 | goto fdput_fd; | ||
1182 | } | ||
1183 | |||
1184 | switch (api_domain) { | ||
1185 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1186 | dev->mclk_target_mhz = target_mhz; | ||
1187 | break; | ||
1188 | |||
1189 | case NVGPU_GPU_CLK_DOMAIN_GPC2CLK: | ||
1190 | dev->gpc2clk_target_mhz = target_mhz; | ||
1191 | break; | ||
1192 | |||
1193 | default: | ||
1194 | err = -EINVAL; | ||
1195 | } | ||
1196 | |||
1197 | fdput_fd: | ||
1198 | fdput(fd); | ||
1199 | return err; | ||
1200 | } | ||
1201 | |||
1202 | int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session, | ||
1203 | u32 api_domain, u16 *freq_mhz) | ||
1204 | { | ||
1205 | int err = 0; | ||
1206 | struct nvgpu_clk_arb_target *target; | ||
1207 | |||
1208 | do { | ||
1209 | target = ACCESS_ONCE(session->target); | ||
1210 | /* no reordering of this pointer */ | ||
1211 | smp_rmb(); | ||
1212 | |||
1213 | switch (api_domain) { | ||
1214 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1215 | *freq_mhz = target->mclk; | ||
1216 | break; | ||
1217 | |||
1218 | case NVGPU_GPU_CLK_DOMAIN_GPC2CLK: | ||
1219 | *freq_mhz = target->gpc2clk; | ||
1220 | break; | ||
1221 | |||
1222 | default: | ||
1223 | *freq_mhz = 0; | ||
1224 | err = -EINVAL; | ||
1225 | } | ||
1226 | } while (target != ACCESS_ONCE(session->target)); | ||
1227 | return err; | ||
1228 | } | ||
1229 | |||
1230 | int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g, | ||
1231 | u32 api_domain, u16 *freq_mhz) | ||
1232 | { | ||
1233 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1234 | int err = 0; | ||
1235 | struct nvgpu_clk_arb_target *actual; | ||
1236 | |||
1237 | do { | ||
1238 | actual = ACCESS_ONCE(arb->actual); | ||
1239 | /* no reordering of this pointer */ | ||
1240 | smp_rmb(); | ||
1241 | |||
1242 | switch (api_domain) { | ||
1243 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1244 | *freq_mhz = actual->mclk; | ||
1245 | break; | ||
1246 | |||
1247 | case NVGPU_GPU_CLK_DOMAIN_GPC2CLK: | ||
1248 | *freq_mhz = actual->gpc2clk; | ||
1249 | break; | ||
1250 | |||
1251 | default: | ||
1252 | *freq_mhz = 0; | ||
1253 | err = -EINVAL; | ||
1254 | } | ||
1255 | } while (actual != ACCESS_ONCE(arb->actual)); | ||
1256 | return err; | ||
1257 | } | ||
1258 | |||
1259 | int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g, | ||
1260 | u32 api_domain, u16 *freq_mhz) | ||
1261 | { | ||
1262 | |||
1263 | *freq_mhz = g->ops.clk.get_rate(g, api_domain); | ||
1264 | return 0; | ||
1265 | } | ||
1266 | |||
1267 | int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain, | ||
1268 | u16 *min_mhz, u16 *max_mhz) | ||
1269 | { | ||
1270 | return g->ops.clk_arb.get_arbiter_clk_range(g, api_domain, | ||
1271 | min_mhz, max_mhz); | ||
1272 | } | ||
1273 | |||
1274 | u32 nvgpu_clk_arb_get_arbiter_clk_domains(struct gk20a *g) | ||
1275 | { | ||
1276 | return g->ops.clk_arb.get_arbiter_clk_domains(g); | ||
1277 | } | ||
1278 | |||
1279 | int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g, | ||
1280 | u32 api_domain, u32 *max_points, u16 *fpoints) | ||
1281 | { | ||
1282 | return (int)clk_domain_get_f_points(g, api_domain, max_points, fpoints); | ||
1283 | } | ||
1284 | |||
1285 | static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb, | ||
1286 | u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk, | ||
1287 | u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram) | ||
1288 | { | ||
1289 | u16 gpc2clk_target, mclk_target; | ||
1290 | u32 gpc2clk_voltuv, gpc2clk_voltuv_sram; | ||
1291 | u32 mclk_voltuv, mclk_voltuv_sram; | ||
1292 | u32 pstate = VF_POINT_INVALID_PSTATE; | ||
1293 | struct nvgpu_clk_vf_table *table; | ||
1294 | u32 index, index_mclk; | ||
1295 | struct nvgpu_clk_vf_point *mclk_vf = NULL; | ||
1296 | |||
1297 | do { | ||
1298 | gpc2clk_target = *gpc2clk; | ||
1299 | mclk_target = *mclk; | ||
1300 | gpc2clk_voltuv = 0; | ||
1301 | gpc2clk_voltuv_sram = 0; | ||
1302 | mclk_voltuv = 0; | ||
1303 | mclk_voltuv_sram = 0; | ||
1304 | |||
1305 | table = ACCESS_ONCE(arb->current_vf_table); | ||
1306 | /* pointer to table can be updated by callback */ | ||
1307 | smp_rmb(); | ||
1308 | |||
1309 | if (!table) | ||
1310 | continue; | ||
1311 | if ((!table->gpc2clk_num_points) || (!table->mclk_num_points)) { | ||
1312 | gk20a_err(dev_from_gk20a(arb->g), "found empty table"); | ||
1313 | goto find_exit; | ||
1314 | } | ||
1315 | /* First we check MCLK to find out which PSTATE we are | ||
1316 | * are requesting, and from there try to find the minimum | ||
1317 | * GPC2CLK on the same PSTATE that satisfies the request. | ||
1318 | * If no GPC2CLK can be found, then we need to up the PSTATE | ||
1319 | */ | ||
1320 | |||
1321 | recalculate_vf_point: | ||
1322 | for (index = 0; index < table->mclk_num_points; index++) { | ||
1323 | if (table->mclk_points[index].mem_mhz >= mclk_target) { | ||
1324 | mclk_vf = &table->mclk_points[index]; | ||
1325 | break; | ||
1326 | } | ||
1327 | } | ||
1328 | if (index == table->mclk_num_points) { | ||
1329 | mclk_vf = &table->mclk_points[index-1]; | ||
1330 | index = table->mclk_num_points - 1; | ||
1331 | } | ||
1332 | index_mclk = index; | ||
1333 | |||
1334 | /* round up the freq requests */ | ||
1335 | for (index = 0; index < table->gpc2clk_num_points; index++) { | ||
1336 | pstate = VF_POINT_COMMON_PSTATE( | ||
1337 | &table->gpc2clk_points[index], mclk_vf); | ||
1338 | |||
1339 | if ((table->gpc2clk_points[index].gpc_mhz >= | ||
1340 | gpc2clk_target) && | ||
1341 | (pstate != VF_POINT_INVALID_PSTATE)){ | ||
1342 | gpc2clk_target = | ||
1343 | table->gpc2clk_points[index].gpc_mhz; | ||
1344 | *sys2clk = | ||
1345 | table->gpc2clk_points[index].sys_mhz; | ||
1346 | *xbar2clk = | ||
1347 | table->gpc2clk_points[index].xbar_mhz; | ||
1348 | |||
1349 | gpc2clk_voltuv = | ||
1350 | table->gpc2clk_points[index].uvolt; | ||
1351 | gpc2clk_voltuv_sram = | ||
1352 | table->gpc2clk_points[index].uvolt_sram; | ||
1353 | break; | ||
1354 | } | ||
1355 | } | ||
1356 | |||
1357 | if (index == table->gpc2clk_num_points) { | ||
1358 | pstate = VF_POINT_COMMON_PSTATE( | ||
1359 | &table->gpc2clk_points[index-1], mclk_vf); | ||
1360 | if (pstate != VF_POINT_INVALID_PSTATE) { | ||
1361 | gpc2clk_target = | ||
1362 | table->gpc2clk_points[index-1].gpc_mhz; | ||
1363 | *sys2clk = | ||
1364 | table->gpc2clk_points[index-1].sys_mhz; | ||
1365 | *xbar2clk = | ||
1366 | table->gpc2clk_points[index-1].xbar_mhz; | ||
1367 | |||
1368 | gpc2clk_voltuv = | ||
1369 | table->gpc2clk_points[index-1].uvolt; | ||
1370 | gpc2clk_voltuv_sram = | ||
1371 | table->gpc2clk_points[index-1]. | ||
1372 | uvolt_sram; | ||
1373 | } else if (index_mclk >= table->mclk_num_points - 1) { | ||
1374 | /* There is no available combination of MCLK | ||
1375 | * and GPC2CLK, we need to fail this | ||
1376 | */ | ||
1377 | gpc2clk_target = 0; | ||
1378 | mclk_target = 0; | ||
1379 | pstate = VF_POINT_INVALID_PSTATE; | ||
1380 | goto find_exit; | ||
1381 | } else { | ||
1382 | /* recalculate with higher PSTATE */ | ||
1383 | gpc2clk_target = *gpc2clk; | ||
1384 | mclk_target = table->mclk_points[index_mclk+1]. | ||
1385 | mem_mhz; | ||
1386 | goto recalculate_vf_point; | ||
1387 | } | ||
1388 | } | ||
1389 | |||
1390 | mclk_target = mclk_vf->mem_mhz; | ||
1391 | mclk_voltuv = mclk_vf->uvolt; | ||
1392 | mclk_voltuv_sram = mclk_vf->uvolt_sram; | ||
1393 | |||
1394 | } while (!table || | ||
1395 | (ACCESS_ONCE(arb->current_vf_table) != table)); | ||
1396 | |||
1397 | find_exit: | ||
1398 | *voltuv = gpc2clk_voltuv > mclk_voltuv ? gpc2clk_voltuv : mclk_voltuv; | ||
1399 | *voltuv_sram = gpc2clk_voltuv_sram > mclk_voltuv_sram ? | ||
1400 | gpc2clk_voltuv_sram : mclk_voltuv_sram; | ||
1401 | /* noise unaware vmin */ | ||
1402 | *nuvmin = mclk_voltuv; | ||
1403 | *nuvmin_sram = mclk_voltuv_sram; | ||
1404 | *gpc2clk = gpc2clk_target; | ||
1405 | *mclk = mclk_target; | ||
1406 | return pstate; | ||
1407 | } | ||
1408 | |||
1409 | /* This function is inherently unsafe to call while arbiter is running | ||
1410 | * arbiter must be blocked before calling this function */ | ||
1411 | int nvgpu_clk_arb_get_current_pstate(struct gk20a *g) | ||
1412 | { | ||
1413 | return ACCESS_ONCE(g->clk_arb->actual->pstate); | ||
1414 | } | ||
1415 | |||
1416 | static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target, | ||
1417 | u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv, | ||
1418 | u32 voltuv_sram) | ||
1419 | { | ||
1420 | struct set_fll_clk fllclk; | ||
1421 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1422 | int status; | ||
1423 | |||
1424 | fllclk.gpc2clkmhz = gpc2clk_target; | ||
1425 | fllclk.sys2clkmhz = sys2clk_target; | ||
1426 | fllclk.xbar2clkmhz = xbar2clk_target; | ||
1427 | |||
1428 | fllclk.voltuv = voltuv; | ||
1429 | |||
1430 | /* if voltage ascends we do: | ||
1431 | * (1) FLL change | ||
1432 | * (2) Voltage change | ||
1433 | * (3) MCLK change | ||
1434 | * If it goes down | ||
1435 | * (1) MCLK change | ||
1436 | * (2) Voltage change | ||
1437 | * (3) FLL change | ||
1438 | */ | ||
1439 | |||
1440 | /* descending */ | ||
1441 | if (voltuv < arb->voltuv_actual) { | ||
1442 | status = g->clk_pmu.clk_mclk.change(g, mclk_target); | ||
1443 | if (status < 0) | ||
1444 | return status; | ||
1445 | |||
1446 | status = volt_set_voltage(g, voltuv, voltuv_sram); | ||
1447 | if (status < 0) | ||
1448 | return status; | ||
1449 | |||
1450 | status = clk_set_fll_clks(g, &fllclk); | ||
1451 | if (status < 0) | ||
1452 | return status; | ||
1453 | } else { | ||
1454 | status = clk_set_fll_clks(g, &fllclk); | ||
1455 | if (status < 0) | ||
1456 | return status; | ||
1457 | |||
1458 | status = volt_set_voltage(g, voltuv, voltuv_sram); | ||
1459 | if (status < 0) | ||
1460 | return status; | ||
1461 | |||
1462 | status = g->clk_pmu.clk_mclk.change(g, mclk_target); | ||
1463 | if (status < 0) | ||
1464 | return status; | ||
1465 | } | ||
1466 | |||
1467 | return 0; | ||
1468 | } | ||
1469 | |||
1470 | void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock) | ||
1471 | { | ||
1472 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1473 | |||
1474 | if (lock) | ||
1475 | mutex_lock(&arb->pstate_lock); | ||
1476 | else | ||
1477 | mutex_unlock(&arb->pstate_lock); | ||
1478 | |||
1479 | } | ||
1480 | |||
1481 | #ifdef CONFIG_DEBUG_FS | ||
1482 | static int nvgpu_clk_arb_stats_show(struct seq_file *s, void *unused) | ||
1483 | { | ||
1484 | struct gk20a *g = s->private; | ||
1485 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1486 | struct nvgpu_clk_arb_debug *debug; | ||
1487 | |||
1488 | u64 num; | ||
1489 | s64 tmp, avg, std, max, min; | ||
1490 | |||
1491 | debug = ACCESS_ONCE(arb->debug); | ||
1492 | /* Make copy of structure and ensure no reordering */ | ||
1493 | smp_rmb(); | ||
1494 | if (!debug) | ||
1495 | return -EINVAL; | ||
1496 | |||
1497 | std = debug->switch_std; | ||
1498 | avg = debug->switch_avg; | ||
1499 | max = debug->switch_max; | ||
1500 | min = debug->switch_min; | ||
1501 | num = debug->switch_num; | ||
1502 | |||
1503 | tmp = std; | ||
1504 | do_div(tmp, num); | ||
1505 | seq_printf(s, "Number of transitions: %lld\n", | ||
1506 | num); | ||
1507 | seq_printf(s, "max / min : %lld / %lld usec\n", | ||
1508 | max, min); | ||
1509 | seq_printf(s, "avg / std : %lld / %ld usec\n", | ||
1510 | avg, int_sqrt(tmp)); | ||
1511 | |||
1512 | return 0; | ||
1513 | } | ||
1514 | |||
1515 | static int nvgpu_clk_arb_stats_open(struct inode *inode, struct file *file) | ||
1516 | { | ||
1517 | return single_open(file, nvgpu_clk_arb_stats_show, inode->i_private); | ||
1518 | } | ||
1519 | |||
1520 | static const struct file_operations nvgpu_clk_arb_stats_fops = { | ||
1521 | .open = nvgpu_clk_arb_stats_open, | ||
1522 | .read = seq_read, | ||
1523 | .llseek = seq_lseek, | ||
1524 | .release = single_release, | ||
1525 | }; | ||
1526 | |||
1527 | |||
1528 | static int nvgpu_clk_arb_debugfs_init(struct gk20a *g) | ||
1529 | { | ||
1530 | struct gk20a_platform *platform = dev_get_drvdata(g->dev); | ||
1531 | |||
1532 | struct dentry *gpu_root = platform->debugfs; | ||
1533 | struct dentry *d; | ||
1534 | |||
1535 | gk20a_dbg(gpu_dbg_info, "g=%p", g); | ||
1536 | |||
1537 | d = debugfs_create_file( | ||
1538 | "arb_stats", | ||
1539 | S_IRUGO, | ||
1540 | gpu_root, | ||
1541 | g, | ||
1542 | &nvgpu_clk_arb_stats_fops); | ||
1543 | if (!d) | ||
1544 | return -ENOMEM; | ||
1545 | |||
1546 | return 0; | ||
1547 | } | ||
1548 | #endif | ||