diff options
author | Mahantesh Kumbar <mkumbar@nvidia.com> | 2017-10-20 03:04:00 -0400 |
---|---|---|
committer | mobile promotions <svcmobile_promotions@nvidia.com> | 2017-10-25 20:29:36 -0400 |
commit | 0dcf0ede812aa55aa106a5e6c2f86216fcbfd5e0 (patch) | |
tree | 6414aa6c9966e98b7c58a9700a8e76abe1f93999 /drivers/gpu/nvgpu/clk/clk_arb.c | |
parent | c79112f3b1e2a428603e06486bd3cea83942c14e (diff) |
gpu: nvgpu: move clk_arb to linux specific
- Clock arbiter has lot of linux dependent code
so moved clk_arb.c to common/linux folder &
clk_arb.h to include/nvgpu/clk_arb.h, this move
helps to unblock QNX.
- QNX must implement functions present
under clk_arb.h as needed.
JIRA NVGPU-33
Change-Id: I38369fafda9c2cb9ba2175b3e530e40d0c746601
Signed-off-by: Mahantesh Kumbar <mkumbar@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1582473
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/clk/clk_arb.c')
-rw-r--r-- | drivers/gpu/nvgpu/clk/clk_arb.c | 2111 |
1 files changed, 0 insertions, 2111 deletions
diff --git a/drivers/gpu/nvgpu/clk/clk_arb.c b/drivers/gpu/nvgpu/clk/clk_arb.c deleted file mode 100644 index 937d47db..00000000 --- a/drivers/gpu/nvgpu/clk/clk_arb.c +++ /dev/null | |||
@@ -1,2111 +0,0 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | |||
24 | #include <linux/cdev.h> | ||
25 | #include <linux/file.h> | ||
26 | #include <linux/anon_inodes.h> | ||
27 | #include <linux/rculist.h> | ||
28 | #include <linux/llist.h> | ||
29 | #include <linux/uaccess.h> | ||
30 | #include <linux/poll.h> | ||
31 | #ifdef CONFIG_DEBUG_FS | ||
32 | #include <linux/debugfs.h> | ||
33 | #endif | ||
34 | |||
35 | #include <nvgpu/bitops.h> | ||
36 | #include <nvgpu/lock.h> | ||
37 | #include <nvgpu/kmem.h> | ||
38 | #include <nvgpu/atomic.h> | ||
39 | #include <nvgpu/bug.h> | ||
40 | #include <nvgpu/kref.h> | ||
41 | #include <nvgpu/log.h> | ||
42 | #include <nvgpu/barrier.h> | ||
43 | #include <nvgpu/cond.h> | ||
44 | |||
45 | #include "gk20a/gk20a.h" | ||
46 | #include "clk/clk_arb.h" | ||
47 | |||
48 | #ifdef CONFIG_DEBUG_FS | ||
49 | #include "common/linux/os_linux.h" | ||
50 | #endif | ||
51 | |||
52 | #define MAX_F_POINTS 256 | ||
53 | #define DEFAULT_EVENT_NUMBER 32 | ||
54 | |||
55 | struct nvgpu_clk_dev; | ||
56 | struct nvgpu_clk_arb_target; | ||
57 | struct nvgpu_clk_notification_queue; | ||
58 | |||
59 | #ifdef CONFIG_DEBUG_FS | ||
60 | static int nvgpu_clk_arb_debugfs_init(struct gk20a *g); | ||
61 | #endif | ||
62 | |||
63 | static int nvgpu_clk_arb_release_event_dev(struct inode *inode, | ||
64 | struct file *filp); | ||
65 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, | ||
66 | struct file *filp); | ||
67 | static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait); | ||
68 | static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf, | ||
69 | size_t size, loff_t *off); | ||
70 | |||
71 | static long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd, | ||
72 | unsigned long arg); | ||
73 | |||
74 | static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work); | ||
75 | static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work); | ||
76 | static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb); | ||
77 | static void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount); | ||
78 | static void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount); | ||
79 | static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target, | ||
80 | u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv, | ||
81 | u32 voltuv_sram); | ||
82 | static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb, | ||
83 | u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk, | ||
84 | u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram); | ||
85 | static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, | ||
86 | struct nvgpu_clk_arb_target *target, | ||
87 | u32 alarm_mask); | ||
88 | static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm); | ||
89 | static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm); | ||
90 | |||
91 | static void nvgpu_clk_arb_queue_notification(struct gk20a *g, | ||
92 | struct nvgpu_clk_notification_queue *queue, | ||
93 | u32 alarm_mask); | ||
94 | static int nvgpu_clk_notification_queue_alloc(struct gk20a *g, | ||
95 | struct nvgpu_clk_notification_queue *queue, | ||
96 | size_t events_number); | ||
97 | |||
98 | static void nvgpu_clk_notification_queue_free(struct gk20a *g, | ||
99 | struct nvgpu_clk_notification_queue *queue); | ||
100 | |||
101 | #define VF_POINT_INVALID_PSTATE ~0U | ||
102 | #define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b))) | ||
103 | #define VF_POINT_GET_PSTATE(a) (((a)->pstates) ?\ | ||
104 | __fls((a)->pstates) :\ | ||
105 | VF_POINT_INVALID_PSTATE) | ||
106 | #define VF_POINT_COMMON_PSTATE(a, b) (((a)->pstates & (b)->pstates) ?\ | ||
107 | __fls((a)->pstates & (b)->pstates) :\ | ||
108 | VF_POINT_INVALID_PSTATE) | ||
109 | |||
110 | /* Local Alarms */ | ||
111 | #define EVENT(alarm) (0x1UL << NVGPU_GPU_EVENT_##alarm) | ||
112 | |||
113 | #define LOCAL_ALARM_MASK (EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE) | \ | ||
114 | EVENT(VF_UPDATE)) | ||
115 | |||
116 | #define _WRAPGTEQ(a, b) ((a-b) > 0) | ||
117 | |||
118 | struct nvgpu_clk_notification { | ||
119 | u32 notification; | ||
120 | u64 timestamp; | ||
121 | }; | ||
122 | |||
123 | struct nvgpu_clk_notification_queue { | ||
124 | u32 size; | ||
125 | nvgpu_atomic_t head; | ||
126 | nvgpu_atomic_t tail; | ||
127 | struct nvgpu_clk_notification *notifications; | ||
128 | }; | ||
129 | |||
130 | struct nvgpu_clk_vf_point { | ||
131 | u16 pstates; | ||
132 | union { | ||
133 | struct { | ||
134 | u16 gpc_mhz; | ||
135 | u16 sys_mhz; | ||
136 | u16 xbar_mhz; | ||
137 | }; | ||
138 | u16 mem_mhz; | ||
139 | }; | ||
140 | u32 uvolt; | ||
141 | u32 uvolt_sram; | ||
142 | }; | ||
143 | |||
144 | struct nvgpu_clk_vf_table { | ||
145 | u32 mclk_num_points; | ||
146 | struct nvgpu_clk_vf_point *mclk_points; | ||
147 | u32 gpc2clk_num_points; | ||
148 | struct nvgpu_clk_vf_point *gpc2clk_points; | ||
149 | }; | ||
150 | #ifdef CONFIG_DEBUG_FS | ||
151 | struct nvgpu_clk_arb_debug { | ||
152 | s64 switch_max; | ||
153 | s64 switch_min; | ||
154 | u64 switch_num; | ||
155 | s64 switch_avg; | ||
156 | s64 switch_std; | ||
157 | }; | ||
158 | #endif | ||
159 | |||
160 | struct nvgpu_clk_arb_target { | ||
161 | u16 mclk; | ||
162 | u16 gpc2clk; | ||
163 | u32 pstate; | ||
164 | }; | ||
165 | |||
166 | struct nvgpu_clk_arb { | ||
167 | struct nvgpu_spinlock sessions_lock; | ||
168 | struct nvgpu_spinlock users_lock; | ||
169 | |||
170 | struct nvgpu_mutex pstate_lock; | ||
171 | struct list_head users; | ||
172 | struct list_head sessions; | ||
173 | struct llist_head requests; | ||
174 | |||
175 | struct gk20a *g; | ||
176 | int status; | ||
177 | |||
178 | struct nvgpu_clk_arb_target actual_pool[2]; | ||
179 | struct nvgpu_clk_arb_target *actual; | ||
180 | |||
181 | u16 gpc2clk_default_mhz; | ||
182 | u16 mclk_default_mhz; | ||
183 | u32 voltuv_actual; | ||
184 | |||
185 | u16 gpc2clk_min, gpc2clk_max; | ||
186 | u16 mclk_min, mclk_max; | ||
187 | |||
188 | struct work_struct update_fn_work; | ||
189 | struct workqueue_struct *update_work_queue; | ||
190 | struct work_struct vf_table_fn_work; | ||
191 | struct workqueue_struct *vf_table_work_queue; | ||
192 | |||
193 | struct nvgpu_cond request_wq; | ||
194 | |||
195 | struct nvgpu_clk_vf_table *current_vf_table; | ||
196 | struct nvgpu_clk_vf_table vf_table_pool[2]; | ||
197 | u32 vf_table_index; | ||
198 | |||
199 | u16 *mclk_f_points; | ||
200 | nvgpu_atomic_t req_nr; | ||
201 | |||
202 | u32 mclk_f_numpoints; | ||
203 | u16 *gpc2clk_f_points; | ||
204 | u32 gpc2clk_f_numpoints; | ||
205 | |||
206 | nvgpu_atomic64_t alarm_mask; | ||
207 | struct nvgpu_clk_notification_queue notification_queue; | ||
208 | |||
209 | #ifdef CONFIG_DEBUG_FS | ||
210 | struct nvgpu_clk_arb_debug debug_pool[2]; | ||
211 | struct nvgpu_clk_arb_debug *debug; | ||
212 | bool debugfs_set; | ||
213 | #endif | ||
214 | }; | ||
215 | |||
216 | struct nvgpu_clk_dev { | ||
217 | struct nvgpu_clk_session *session; | ||
218 | union { | ||
219 | struct list_head link; | ||
220 | struct llist_node node; | ||
221 | }; | ||
222 | struct nvgpu_cond readout_wq; | ||
223 | nvgpu_atomic_t poll_mask; | ||
224 | u16 gpc2clk_target_mhz; | ||
225 | u16 mclk_target_mhz; | ||
226 | u32 alarms_reported; | ||
227 | nvgpu_atomic_t enabled_mask; | ||
228 | struct nvgpu_clk_notification_queue queue; | ||
229 | u32 arb_queue_head; | ||
230 | struct nvgpu_ref refcount; | ||
231 | }; | ||
232 | |||
233 | struct nvgpu_clk_session { | ||
234 | bool zombie; | ||
235 | struct gk20a *g; | ||
236 | struct nvgpu_ref refcount; | ||
237 | struct list_head link; | ||
238 | struct llist_head targets; | ||
239 | |||
240 | struct nvgpu_clk_arb_target target_pool[2]; | ||
241 | struct nvgpu_clk_arb_target *target; | ||
242 | }; | ||
243 | |||
244 | static const struct file_operations completion_dev_ops = { | ||
245 | .owner = THIS_MODULE, | ||
246 | .release = nvgpu_clk_arb_release_completion_dev, | ||
247 | .poll = nvgpu_clk_arb_poll_dev, | ||
248 | }; | ||
249 | |||
250 | static const struct file_operations event_dev_ops = { | ||
251 | .owner = THIS_MODULE, | ||
252 | .release = nvgpu_clk_arb_release_event_dev, | ||
253 | .poll = nvgpu_clk_arb_poll_dev, | ||
254 | .read = nvgpu_clk_arb_read_event_dev, | ||
255 | #ifdef CONFIG_COMPAT | ||
256 | .compat_ioctl = nvgpu_clk_arb_ioctl_event_dev, | ||
257 | #endif | ||
258 | .unlocked_ioctl = nvgpu_clk_arb_ioctl_event_dev, | ||
259 | }; | ||
260 | |||
261 | static int nvgpu_clk_notification_queue_alloc(struct gk20a *g, | ||
262 | struct nvgpu_clk_notification_queue *queue, | ||
263 | size_t events_number) { | ||
264 | queue->notifications = nvgpu_kcalloc(g, events_number, | ||
265 | sizeof(struct nvgpu_clk_notification)); | ||
266 | if (!queue->notifications) | ||
267 | return -ENOMEM; | ||
268 | queue->size = events_number; | ||
269 | |||
270 | nvgpu_atomic_set(&queue->head, 0); | ||
271 | nvgpu_atomic_set(&queue->tail, 0); | ||
272 | |||
273 | return 0; | ||
274 | } | ||
275 | |||
276 | static void nvgpu_clk_notification_queue_free(struct gk20a *g, | ||
277 | struct nvgpu_clk_notification_queue *queue) { | ||
278 | nvgpu_kfree(g, queue->notifications); | ||
279 | queue->size = 0; | ||
280 | nvgpu_atomic_set(&queue->head, 0); | ||
281 | nvgpu_atomic_set(&queue->tail, 0); | ||
282 | } | ||
283 | |||
284 | int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | ||
285 | { | ||
286 | struct nvgpu_clk_arb *arb; | ||
287 | u16 default_mhz; | ||
288 | int err; | ||
289 | int index; | ||
290 | struct nvgpu_clk_vf_table *table; | ||
291 | |||
292 | gk20a_dbg_fn(""); | ||
293 | |||
294 | if (!g->ops.clk_arb.get_arbiter_clk_domains) | ||
295 | return 0; | ||
296 | |||
297 | arb = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_arb)); | ||
298 | if (!arb) | ||
299 | return -ENOMEM; | ||
300 | |||
301 | err = nvgpu_mutex_init(&arb->pstate_lock); | ||
302 | if (err) | ||
303 | goto mutex_fail; | ||
304 | nvgpu_spinlock_init(&arb->sessions_lock); | ||
305 | nvgpu_spinlock_init(&arb->users_lock); | ||
306 | |||
307 | arb->mclk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16)); | ||
308 | if (!arb->mclk_f_points) { | ||
309 | err = -ENOMEM; | ||
310 | goto init_fail; | ||
311 | } | ||
312 | |||
313 | arb->gpc2clk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16)); | ||
314 | if (!arb->gpc2clk_f_points) { | ||
315 | err = -ENOMEM; | ||
316 | goto init_fail; | ||
317 | } | ||
318 | |||
319 | for (index = 0; index < 2; index++) { | ||
320 | table = &arb->vf_table_pool[index]; | ||
321 | table->gpc2clk_num_points = MAX_F_POINTS; | ||
322 | table->mclk_num_points = MAX_F_POINTS; | ||
323 | |||
324 | table->gpc2clk_points = nvgpu_kcalloc(g, MAX_F_POINTS, | ||
325 | sizeof(struct nvgpu_clk_vf_point)); | ||
326 | if (!table->gpc2clk_points) { | ||
327 | err = -ENOMEM; | ||
328 | goto init_fail; | ||
329 | } | ||
330 | |||
331 | |||
332 | table->mclk_points = nvgpu_kcalloc(g, MAX_F_POINTS, | ||
333 | sizeof(struct nvgpu_clk_vf_point)); | ||
334 | if (!table->mclk_points) { | ||
335 | err = -ENOMEM; | ||
336 | goto init_fail; | ||
337 | } | ||
338 | } | ||
339 | |||
340 | g->clk_arb = arb; | ||
341 | arb->g = g; | ||
342 | |||
343 | err = g->ops.clk_arb.get_arbiter_clk_default(g, | ||
344 | CTRL_CLK_DOMAIN_MCLK, &default_mhz); | ||
345 | if (err < 0) { | ||
346 | err = -EINVAL; | ||
347 | goto init_fail; | ||
348 | } | ||
349 | |||
350 | arb->mclk_default_mhz = default_mhz; | ||
351 | |||
352 | err = g->ops.clk_arb.get_arbiter_clk_default(g, | ||
353 | CTRL_CLK_DOMAIN_GPC2CLK, &default_mhz); | ||
354 | if (err < 0) { | ||
355 | err = -EINVAL; | ||
356 | goto init_fail; | ||
357 | } | ||
358 | |||
359 | arb->gpc2clk_default_mhz = default_mhz; | ||
360 | |||
361 | arb->actual = &arb->actual_pool[0]; | ||
362 | |||
363 | nvgpu_atomic_set(&arb->req_nr, 0); | ||
364 | |||
365 | nvgpu_atomic64_set(&arb->alarm_mask, 0); | ||
366 | err = nvgpu_clk_notification_queue_alloc(g, &arb->notification_queue, | ||
367 | DEFAULT_EVENT_NUMBER); | ||
368 | if (err < 0) | ||
369 | goto init_fail; | ||
370 | |||
371 | INIT_LIST_HEAD_RCU(&arb->users); | ||
372 | INIT_LIST_HEAD_RCU(&arb->sessions); | ||
373 | init_llist_head(&arb->requests); | ||
374 | |||
375 | nvgpu_cond_init(&arb->request_wq); | ||
376 | arb->vf_table_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1, | ||
377 | "vf_table_update"); | ||
378 | arb->update_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1, | ||
379 | "arbiter_update"); | ||
380 | |||
381 | |||
382 | INIT_WORK(&arb->vf_table_fn_work, nvgpu_clk_arb_run_vf_table_cb); | ||
383 | |||
384 | INIT_WORK(&arb->update_fn_work, nvgpu_clk_arb_run_arbiter_cb); | ||
385 | |||
386 | #ifdef CONFIG_DEBUG_FS | ||
387 | arb->debug = &arb->debug_pool[0]; | ||
388 | |||
389 | if (!arb->debugfs_set) { | ||
390 | if (nvgpu_clk_arb_debugfs_init(g)) | ||
391 | arb->debugfs_set = true; | ||
392 | } | ||
393 | #endif | ||
394 | err = clk_vf_point_cache(g); | ||
395 | if (err < 0) | ||
396 | goto init_fail; | ||
397 | |||
398 | err = nvgpu_clk_arb_update_vf_table(arb); | ||
399 | if (err < 0) | ||
400 | goto init_fail; | ||
401 | do { | ||
402 | /* Check that first run is completed */ | ||
403 | nvgpu_smp_mb(); | ||
404 | NVGPU_COND_WAIT_INTERRUPTIBLE(&arb->request_wq, | ||
405 | nvgpu_atomic_read(&arb->req_nr), 0); | ||
406 | } while (!nvgpu_atomic_read(&arb->req_nr)); | ||
407 | |||
408 | |||
409 | return arb->status; | ||
410 | |||
411 | init_fail: | ||
412 | nvgpu_kfree(g, arb->gpc2clk_f_points); | ||
413 | nvgpu_kfree(g, arb->mclk_f_points); | ||
414 | |||
415 | for (index = 0; index < 2; index++) { | ||
416 | nvgpu_kfree(g, arb->vf_table_pool[index].gpc2clk_points); | ||
417 | nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points); | ||
418 | } | ||
419 | |||
420 | nvgpu_mutex_destroy(&arb->pstate_lock); | ||
421 | |||
422 | mutex_fail: | ||
423 | nvgpu_kfree(g, arb); | ||
424 | |||
425 | return err; | ||
426 | } | ||
427 | |||
428 | void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm) | ||
429 | { | ||
430 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
431 | |||
432 | nvgpu_clk_arb_set_global_alarm(g, alarm); | ||
433 | if (arb->update_work_queue) | ||
434 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
435 | } | ||
436 | |||
437 | static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm) | ||
438 | { | ||
439 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
440 | |||
441 | u64 current_mask; | ||
442 | u32 refcnt; | ||
443 | u32 alarm_mask; | ||
444 | u64 new_mask; | ||
445 | |||
446 | do { | ||
447 | current_mask = nvgpu_atomic64_read(&arb->alarm_mask); | ||
448 | /* atomic operations are strong so they do not need masks */ | ||
449 | |||
450 | refcnt = ((u32) (current_mask >> 32)) + 1; | ||
451 | alarm_mask = (u32) (current_mask & ~alarm); | ||
452 | new_mask = ((u64) refcnt << 32) | alarm_mask; | ||
453 | |||
454 | } while (unlikely(current_mask != | ||
455 | (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask, | ||
456 | current_mask, new_mask))); | ||
457 | } | ||
458 | |||
459 | static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm) | ||
460 | { | ||
461 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
462 | |||
463 | u64 current_mask; | ||
464 | u32 refcnt; | ||
465 | u32 alarm_mask; | ||
466 | u64 new_mask; | ||
467 | |||
468 | do { | ||
469 | current_mask = nvgpu_atomic64_read(&arb->alarm_mask); | ||
470 | /* atomic operations are strong so they do not need masks */ | ||
471 | |||
472 | refcnt = ((u32) (current_mask >> 32)) + 1; | ||
473 | alarm_mask = (u32) (current_mask & ~0) | alarm; | ||
474 | new_mask = ((u64) refcnt << 32) | alarm_mask; | ||
475 | |||
476 | } while (unlikely(current_mask != | ||
477 | (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask, | ||
478 | current_mask, new_mask))); | ||
479 | |||
480 | nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm); | ||
481 | |||
482 | } | ||
483 | |||
484 | void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g) | ||
485 | { | ||
486 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
487 | int index; | ||
488 | |||
489 | if (arb) { | ||
490 | cancel_work_sync(&arb->vf_table_fn_work); | ||
491 | destroy_workqueue(arb->vf_table_work_queue); | ||
492 | arb->vf_table_work_queue = NULL; | ||
493 | |||
494 | cancel_work_sync(&arb->update_fn_work); | ||
495 | destroy_workqueue(arb->update_work_queue); | ||
496 | arb->update_work_queue = NULL; | ||
497 | |||
498 | nvgpu_kfree(g, arb->gpc2clk_f_points); | ||
499 | nvgpu_kfree(g, arb->mclk_f_points); | ||
500 | |||
501 | for (index = 0; index < 2; index++) { | ||
502 | nvgpu_kfree(g, arb->vf_table_pool[index].gpc2clk_points); | ||
503 | nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points); | ||
504 | } | ||
505 | nvgpu_mutex_destroy(&g->clk_arb->pstate_lock); | ||
506 | nvgpu_kfree(g, g->clk_arb); | ||
507 | g->clk_arb = NULL; | ||
508 | } | ||
509 | } | ||
510 | |||
511 | static int nvgpu_clk_arb_install_fd(struct gk20a *g, | ||
512 | struct nvgpu_clk_session *session, | ||
513 | const struct file_operations *fops, | ||
514 | struct nvgpu_clk_dev **_dev) | ||
515 | { | ||
516 | struct file *file; | ||
517 | int fd; | ||
518 | int err; | ||
519 | int status; | ||
520 | char name[64]; | ||
521 | struct nvgpu_clk_dev *dev; | ||
522 | |||
523 | gk20a_dbg_fn(""); | ||
524 | |||
525 | dev = nvgpu_kzalloc(g, sizeof(*dev)); | ||
526 | if (!dev) | ||
527 | return -ENOMEM; | ||
528 | |||
529 | status = nvgpu_clk_notification_queue_alloc(g, &dev->queue, | ||
530 | DEFAULT_EVENT_NUMBER); | ||
531 | if (status < 0) { | ||
532 | err = status; | ||
533 | goto fail; | ||
534 | } | ||
535 | |||
536 | fd = get_unused_fd_flags(O_RDWR); | ||
537 | if (fd < 0) { | ||
538 | err = fd; | ||
539 | goto fail; | ||
540 | } | ||
541 | |||
542 | snprintf(name, sizeof(name), "%s-clk-fd%d", g->name, fd); | ||
543 | file = anon_inode_getfile(name, fops, dev, O_RDWR); | ||
544 | if (IS_ERR(file)) { | ||
545 | err = PTR_ERR(file); | ||
546 | goto fail_fd; | ||
547 | } | ||
548 | |||
549 | fd_install(fd, file); | ||
550 | |||
551 | nvgpu_cond_init(&dev->readout_wq); | ||
552 | |||
553 | nvgpu_atomic_set(&dev->poll_mask, 0); | ||
554 | |||
555 | dev->session = session; | ||
556 | nvgpu_ref_init(&dev->refcount); | ||
557 | |||
558 | nvgpu_ref_get(&session->refcount); | ||
559 | |||
560 | *_dev = dev; | ||
561 | |||
562 | return fd; | ||
563 | |||
564 | fail_fd: | ||
565 | put_unused_fd(fd); | ||
566 | fail: | ||
567 | nvgpu_kfree(g, dev); | ||
568 | |||
569 | return err; | ||
570 | } | ||
571 | |||
572 | int nvgpu_clk_arb_init_session(struct gk20a *g, | ||
573 | struct nvgpu_clk_session **_session) | ||
574 | { | ||
575 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
576 | struct nvgpu_clk_session *session = *(_session); | ||
577 | |||
578 | gk20a_dbg_fn(""); | ||
579 | |||
580 | if (!g->ops.clk_arb.get_arbiter_clk_domains) | ||
581 | return 0; | ||
582 | |||
583 | session = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_session)); | ||
584 | if (!session) | ||
585 | return -ENOMEM; | ||
586 | session->g = g; | ||
587 | |||
588 | nvgpu_ref_init(&session->refcount); | ||
589 | |||
590 | session->zombie = false; | ||
591 | session->target_pool[0].pstate = CTRL_PERF_PSTATE_P8; | ||
592 | /* make sure that the initialization of the pool is visible | ||
593 | * before the update */ | ||
594 | nvgpu_smp_wmb(); | ||
595 | session->target = &session->target_pool[0]; | ||
596 | |||
597 | init_llist_head(&session->targets); | ||
598 | |||
599 | nvgpu_spinlock_acquire(&arb->sessions_lock); | ||
600 | list_add_tail_rcu(&session->link, &arb->sessions); | ||
601 | nvgpu_spinlock_release(&arb->sessions_lock); | ||
602 | |||
603 | *_session = session; | ||
604 | |||
605 | return 0; | ||
606 | } | ||
607 | |||
608 | static void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount) | ||
609 | { | ||
610 | struct nvgpu_clk_dev *dev = container_of(refcount, | ||
611 | struct nvgpu_clk_dev, refcount); | ||
612 | struct nvgpu_clk_session *session = dev->session; | ||
613 | |||
614 | nvgpu_kfree(session->g, dev); | ||
615 | } | ||
616 | |||
617 | static void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount) | ||
618 | { | ||
619 | struct nvgpu_clk_session *session = container_of(refcount, | ||
620 | struct nvgpu_clk_session, refcount); | ||
621 | struct nvgpu_clk_arb *arb = session->g->clk_arb; | ||
622 | struct gk20a *g = session->g; | ||
623 | struct nvgpu_clk_dev *dev, *tmp; | ||
624 | struct llist_node *head; | ||
625 | |||
626 | gk20a_dbg_fn(""); | ||
627 | |||
628 | if (arb) { | ||
629 | nvgpu_spinlock_acquire(&arb->sessions_lock); | ||
630 | list_del_rcu(&session->link); | ||
631 | nvgpu_spinlock_release(&arb->sessions_lock); | ||
632 | } | ||
633 | |||
634 | head = llist_del_all(&session->targets); | ||
635 | llist_for_each_entry_safe(dev, tmp, head, node) { | ||
636 | nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | ||
637 | } | ||
638 | synchronize_rcu(); | ||
639 | nvgpu_kfree(g, session); | ||
640 | } | ||
641 | |||
642 | void nvgpu_clk_arb_release_session(struct gk20a *g, | ||
643 | struct nvgpu_clk_session *session) | ||
644 | { | ||
645 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
646 | |||
647 | gk20a_dbg_fn(""); | ||
648 | |||
649 | session->zombie = true; | ||
650 | nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session); | ||
651 | if (arb && arb->update_work_queue) | ||
652 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
653 | } | ||
654 | |||
655 | int nvgpu_clk_arb_install_event_fd(struct gk20a *g, | ||
656 | struct nvgpu_clk_session *session, int *event_fd, u32 alarm_mask) | ||
657 | { | ||
658 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
659 | struct nvgpu_clk_dev *dev; | ||
660 | int fd; | ||
661 | |||
662 | gk20a_dbg_fn(""); | ||
663 | |||
664 | fd = nvgpu_clk_arb_install_fd(g, session, &event_dev_ops, &dev); | ||
665 | if (fd < 0) | ||
666 | return fd; | ||
667 | |||
668 | /* TODO: alarm mask needs to be set to default value to prevent | ||
669 | * failures of legacy tests. This will be removed when sanity is | ||
670 | * updated | ||
671 | */ | ||
672 | if (alarm_mask) | ||
673 | nvgpu_atomic_set(&dev->enabled_mask, alarm_mask); | ||
674 | else | ||
675 | nvgpu_atomic_set(&dev->enabled_mask, EVENT(VF_UPDATE)); | ||
676 | |||
677 | dev->arb_queue_head = nvgpu_atomic_read(&arb->notification_queue.head); | ||
678 | |||
679 | nvgpu_spinlock_acquire(&arb->users_lock); | ||
680 | list_add_tail_rcu(&dev->link, &arb->users); | ||
681 | nvgpu_spinlock_release(&arb->users_lock); | ||
682 | |||
683 | *event_fd = fd; | ||
684 | |||
685 | return 0; | ||
686 | } | ||
687 | |||
688 | int nvgpu_clk_arb_install_request_fd(struct gk20a *g, | ||
689 | struct nvgpu_clk_session *session, int *request_fd) | ||
690 | { | ||
691 | struct nvgpu_clk_dev *dev; | ||
692 | int fd; | ||
693 | |||
694 | gk20a_dbg_fn(""); | ||
695 | |||
696 | fd = nvgpu_clk_arb_install_fd(g, session, &completion_dev_ops, &dev); | ||
697 | if (fd < 0) | ||
698 | return fd; | ||
699 | |||
700 | *request_fd = fd; | ||
701 | |||
702 | return 0; | ||
703 | } | ||
704 | |||
705 | static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb) | ||
706 | { | ||
707 | struct gk20a *g = arb->g; | ||
708 | struct nvgpu_clk_vf_table *table; | ||
709 | |||
710 | u32 i, j; | ||
711 | int status = -EINVAL; | ||
712 | u32 gpc2clk_voltuv = 0, mclk_voltuv = 0; | ||
713 | u32 gpc2clk_voltuv_sram = 0, mclk_voltuv_sram = 0; | ||
714 | u16 clk_cur; | ||
715 | u32 num_points; | ||
716 | |||
717 | struct clk_set_info *p5_info, *p0_info; | ||
718 | |||
719 | |||
720 | table = NV_ACCESS_ONCE(arb->current_vf_table); | ||
721 | /* make flag visible when all data has resolved in the tables */ | ||
722 | nvgpu_smp_rmb(); | ||
723 | |||
724 | table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] : | ||
725 | &arb->vf_table_pool[0]; | ||
726 | |||
727 | /* Get allowed memory ranges */ | ||
728 | if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
729 | &arb->gpc2clk_min, | ||
730 | &arb->gpc2clk_max) < 0) { | ||
731 | nvgpu_err(g, "failed to fetch GPC2CLK range"); | ||
732 | goto exit_vf_table; | ||
733 | } | ||
734 | if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_MCLK, | ||
735 | &arb->mclk_min, | ||
736 | &arb->mclk_max) < 0) { | ||
737 | nvgpu_err(g, "failed to fetch MCLK range"); | ||
738 | goto exit_vf_table; | ||
739 | } | ||
740 | |||
741 | table->gpc2clk_num_points = MAX_F_POINTS; | ||
742 | table->mclk_num_points = MAX_F_POINTS; | ||
743 | |||
744 | if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
745 | &table->gpc2clk_num_points, arb->gpc2clk_f_points)) { | ||
746 | nvgpu_err(g, "failed to fetch GPC2CLK frequency points"); | ||
747 | goto exit_vf_table; | ||
748 | } | ||
749 | |||
750 | if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_MCLK, | ||
751 | &table->mclk_num_points, arb->mclk_f_points)) { | ||
752 | nvgpu_err(g, "failed to fetch MCLK frequency points"); | ||
753 | goto exit_vf_table; | ||
754 | } | ||
755 | if (!table->mclk_num_points || !table->gpc2clk_num_points) { | ||
756 | nvgpu_err(g, "empty queries to f points mclk %d gpc2clk %d", | ||
757 | table->mclk_num_points, table->gpc2clk_num_points); | ||
758 | status = -EINVAL; | ||
759 | goto exit_vf_table; | ||
760 | } | ||
761 | |||
762 | memset(table->mclk_points, 0, | ||
763 | table->mclk_num_points*sizeof(struct nvgpu_clk_vf_point)); | ||
764 | memset(table->gpc2clk_points, 0, | ||
765 | table->gpc2clk_num_points*sizeof(struct nvgpu_clk_vf_point)); | ||
766 | |||
767 | p5_info = pstate_get_clk_set_info(g, | ||
768 | CTRL_PERF_PSTATE_P5, clkwhich_mclk); | ||
769 | if (!p5_info) { | ||
770 | nvgpu_err(g, "failed to get MCLK P5 info"); | ||
771 | goto exit_vf_table; | ||
772 | } | ||
773 | p0_info = pstate_get_clk_set_info(g, | ||
774 | CTRL_PERF_PSTATE_P0, clkwhich_mclk); | ||
775 | if (!p0_info) { | ||
776 | nvgpu_err(g, "failed to get MCLK P0 info"); | ||
777 | goto exit_vf_table; | ||
778 | } | ||
779 | |||
780 | for (i = 0, j = 0, num_points = 0, clk_cur = 0; | ||
781 | i < table->mclk_num_points; i++) { | ||
782 | |||
783 | if ((arb->mclk_f_points[i] >= arb->mclk_min) && | ||
784 | (arb->mclk_f_points[i] <= arb->mclk_max) && | ||
785 | (arb->mclk_f_points[i] != clk_cur)) { | ||
786 | |||
787 | table->mclk_points[j].mem_mhz = arb->mclk_f_points[i]; | ||
788 | mclk_voltuv = mclk_voltuv_sram = 0; | ||
789 | |||
790 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK, | ||
791 | &table->mclk_points[j].mem_mhz, &mclk_voltuv, | ||
792 | CTRL_VOLT_DOMAIN_LOGIC); | ||
793 | if (status < 0) { | ||
794 | nvgpu_err(g, | ||
795 | "failed to get MCLK LOGIC voltage"); | ||
796 | goto exit_vf_table; | ||
797 | } | ||
798 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK, | ||
799 | &table->mclk_points[j].mem_mhz, | ||
800 | &mclk_voltuv_sram, | ||
801 | CTRL_VOLT_DOMAIN_SRAM); | ||
802 | if (status < 0) { | ||
803 | nvgpu_err(g, "failed to get MCLK SRAM voltage"); | ||
804 | goto exit_vf_table; | ||
805 | } | ||
806 | |||
807 | table->mclk_points[j].uvolt = mclk_voltuv; | ||
808 | table->mclk_points[j].uvolt_sram = mclk_voltuv_sram; | ||
809 | clk_cur = table->mclk_points[j].mem_mhz; | ||
810 | |||
811 | if ((clk_cur >= p5_info->min_mhz) && | ||
812 | (clk_cur <= p5_info->max_mhz)) | ||
813 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
814 | &table->mclk_points[j], | ||
815 | CTRL_PERF_PSTATE_P5); | ||
816 | if ((clk_cur >= p0_info->min_mhz) && | ||
817 | (clk_cur <= p0_info->max_mhz)) | ||
818 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
819 | &table->mclk_points[j], | ||
820 | CTRL_PERF_PSTATE_P0); | ||
821 | |||
822 | j++; | ||
823 | num_points++; | ||
824 | |||
825 | } | ||
826 | } | ||
827 | table->mclk_num_points = num_points; | ||
828 | |||
829 | p5_info = pstate_get_clk_set_info(g, | ||
830 | CTRL_PERF_PSTATE_P5, clkwhich_gpc2clk); | ||
831 | if (!p5_info) { | ||
832 | status = -EINVAL; | ||
833 | nvgpu_err(g, "failed to get GPC2CLK P5 info"); | ||
834 | goto exit_vf_table; | ||
835 | } | ||
836 | |||
837 | p0_info = pstate_get_clk_set_info(g, | ||
838 | CTRL_PERF_PSTATE_P0, clkwhich_gpc2clk); | ||
839 | if (!p0_info) { | ||
840 | status = -EINVAL; | ||
841 | nvgpu_err(g, "failed to get GPC2CLK P0 info"); | ||
842 | goto exit_vf_table; | ||
843 | } | ||
844 | |||
845 | /* GPC2CLK needs to be checked in two passes. The first determines the | ||
846 | * relationships between GPC2CLK, SYS2CLK and XBAR2CLK, while the | ||
847 | * second verifies that the clocks minimum is satisfied and sets | ||
848 | * the voltages | ||
849 | */ | ||
850 | for (i = 0, j = 0, num_points = 0, clk_cur = 0; | ||
851 | i < table->gpc2clk_num_points; i++) { | ||
852 | struct set_fll_clk setfllclk; | ||
853 | |||
854 | if ((arb->gpc2clk_f_points[i] >= arb->gpc2clk_min) && | ||
855 | (arb->gpc2clk_f_points[i] <= arb->gpc2clk_max) && | ||
856 | (arb->gpc2clk_f_points[i] != clk_cur)) { | ||
857 | |||
858 | table->gpc2clk_points[j].gpc_mhz = | ||
859 | arb->gpc2clk_f_points[i]; | ||
860 | setfllclk.gpc2clkmhz = arb->gpc2clk_f_points[i]; | ||
861 | status = clk_get_fll_clks(g, &setfllclk); | ||
862 | if (status < 0) { | ||
863 | nvgpu_err(g, | ||
864 | "failed to get GPC2CLK slave clocks"); | ||
865 | goto exit_vf_table; | ||
866 | } | ||
867 | |||
868 | table->gpc2clk_points[j].sys_mhz = | ||
869 | setfllclk.sys2clkmhz; | ||
870 | table->gpc2clk_points[j].xbar_mhz = | ||
871 | setfllclk.xbar2clkmhz; | ||
872 | |||
873 | clk_cur = table->gpc2clk_points[j].gpc_mhz; | ||
874 | |||
875 | if ((clk_cur >= p5_info->min_mhz) && | ||
876 | (clk_cur <= p5_info->max_mhz)) | ||
877 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
878 | &table->gpc2clk_points[j], | ||
879 | CTRL_PERF_PSTATE_P5); | ||
880 | if ((clk_cur >= p0_info->min_mhz) && | ||
881 | (clk_cur <= p0_info->max_mhz)) | ||
882 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
883 | &table->gpc2clk_points[j], | ||
884 | CTRL_PERF_PSTATE_P0); | ||
885 | |||
886 | j++; | ||
887 | num_points++; | ||
888 | } | ||
889 | } | ||
890 | table->gpc2clk_num_points = num_points; | ||
891 | |||
892 | /* Second pass */ | ||
893 | for (i = 0, j = 0; i < table->gpc2clk_num_points; i++) { | ||
894 | |||
895 | u16 alt_gpc2clk = table->gpc2clk_points[i].gpc_mhz; | ||
896 | gpc2clk_voltuv = gpc2clk_voltuv_sram = 0; | ||
897 | |||
898 | /* Check sysclk */ | ||
899 | p5_info = pstate_get_clk_set_info(g, | ||
900 | VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]), | ||
901 | clkwhich_sys2clk); | ||
902 | if (!p5_info) { | ||
903 | status = -EINVAL; | ||
904 | nvgpu_err(g, "failed to get SYS2CLK P5 info"); | ||
905 | goto exit_vf_table; | ||
906 | } | ||
907 | |||
908 | /* sys2clk below clk min, need to find correct clock */ | ||
909 | if (table->gpc2clk_points[i].sys_mhz < p5_info->min_mhz) { | ||
910 | for (j = i + 1; j < table->gpc2clk_num_points; j++) { | ||
911 | |||
912 | if (table->gpc2clk_points[j].sys_mhz >= | ||
913 | p5_info->min_mhz) { | ||
914 | |||
915 | |||
916 | table->gpc2clk_points[i].sys_mhz = | ||
917 | p5_info->min_mhz; | ||
918 | |||
919 | alt_gpc2clk = alt_gpc2clk < | ||
920 | table->gpc2clk_points[j]. | ||
921 | gpc_mhz ? | ||
922 | table->gpc2clk_points[j]. | ||
923 | gpc_mhz: | ||
924 | alt_gpc2clk; | ||
925 | break; | ||
926 | } | ||
927 | } | ||
928 | /* no VF exists that satisfies condition */ | ||
929 | if (j == table->gpc2clk_num_points) { | ||
930 | nvgpu_err(g, "NO SYS2CLK VF point possible"); | ||
931 | status = -EINVAL; | ||
932 | goto exit_vf_table; | ||
933 | } | ||
934 | } | ||
935 | |||
936 | /* Check xbarclk */ | ||
937 | p5_info = pstate_get_clk_set_info(g, | ||
938 | VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]), | ||
939 | clkwhich_xbar2clk); | ||
940 | if (!p5_info) { | ||
941 | status = -EINVAL; | ||
942 | nvgpu_err(g, "failed to get SYS2CLK P5 info"); | ||
943 | goto exit_vf_table; | ||
944 | } | ||
945 | |||
946 | /* xbar2clk below clk min, need to find correct clock */ | ||
947 | if (table->gpc2clk_points[i].xbar_mhz < p5_info->min_mhz) { | ||
948 | for (j = i; j < table->gpc2clk_num_points; j++) { | ||
949 | if (table->gpc2clk_points[j].xbar_mhz >= | ||
950 | p5_info->min_mhz) { | ||
951 | |||
952 | table->gpc2clk_points[i].xbar_mhz = | ||
953 | p5_info->min_mhz; | ||
954 | |||
955 | alt_gpc2clk = alt_gpc2clk < | ||
956 | table->gpc2clk_points[j]. | ||
957 | gpc_mhz ? | ||
958 | table->gpc2clk_points[j]. | ||
959 | gpc_mhz: | ||
960 | alt_gpc2clk; | ||
961 | break; | ||
962 | } | ||
963 | } | ||
964 | /* no VF exists that satisfies condition */ | ||
965 | if (j == table->gpc2clk_num_points) { | ||
966 | status = -EINVAL; | ||
967 | nvgpu_err(g, "NO XBAR2CLK VF point possible"); | ||
968 | |||
969 | goto exit_vf_table; | ||
970 | } | ||
971 | } | ||
972 | |||
973 | /* Calculate voltages */ | ||
974 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
975 | &alt_gpc2clk, &gpc2clk_voltuv, | ||
976 | CTRL_VOLT_DOMAIN_LOGIC); | ||
977 | if (status < 0) { | ||
978 | nvgpu_err(g, "failed to get GPC2CLK LOGIC voltage"); | ||
979 | goto exit_vf_table; | ||
980 | } | ||
981 | |||
982 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
983 | &alt_gpc2clk, | ||
984 | &gpc2clk_voltuv_sram, | ||
985 | CTRL_VOLT_DOMAIN_SRAM); | ||
986 | if (status < 0) { | ||
987 | nvgpu_err(g, "failed to get GPC2CLK SRAM voltage"); | ||
988 | goto exit_vf_table; | ||
989 | } | ||
990 | |||
991 | table->gpc2clk_points[i].uvolt = gpc2clk_voltuv; | ||
992 | table->gpc2clk_points[i].uvolt_sram = gpc2clk_voltuv_sram; | ||
993 | } | ||
994 | |||
995 | /* make table visible when all data has resolved in the tables */ | ||
996 | nvgpu_smp_wmb(); | ||
997 | xchg(&arb->current_vf_table, table); | ||
998 | |||
999 | exit_vf_table: | ||
1000 | |||
1001 | if (status < 0) | ||
1002 | nvgpu_clk_arb_set_global_alarm(g, | ||
1003 | EVENT(ALARM_VF_TABLE_UPDATE_FAILED)); | ||
1004 | if (arb->update_work_queue) | ||
1005 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
1006 | |||
1007 | return status; | ||
1008 | } | ||
1009 | |||
1010 | void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g) | ||
1011 | { | ||
1012 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1013 | if (arb->vf_table_work_queue) | ||
1014 | queue_work(arb->vf_table_work_queue, &arb->vf_table_fn_work); | ||
1015 | } | ||
1016 | |||
1017 | static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work) | ||
1018 | { | ||
1019 | struct nvgpu_clk_arb *arb = | ||
1020 | container_of(work, struct nvgpu_clk_arb, vf_table_fn_work); | ||
1021 | struct gk20a *g = arb->g; | ||
1022 | u32 err; | ||
1023 | |||
1024 | /* get latest vf curve from pmu */ | ||
1025 | err = clk_vf_point_cache(g); | ||
1026 | if (err) { | ||
1027 | nvgpu_err(g, "failed to cache VF table"); | ||
1028 | nvgpu_clk_arb_set_global_alarm(g, | ||
1029 | EVENT(ALARM_VF_TABLE_UPDATE_FAILED)); | ||
1030 | if (arb->update_work_queue) | ||
1031 | queue_work(arb->update_work_queue, | ||
1032 | &arb->update_fn_work); | ||
1033 | |||
1034 | return; | ||
1035 | } | ||
1036 | nvgpu_clk_arb_update_vf_table(arb); | ||
1037 | } | ||
1038 | |||
1039 | static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | ||
1040 | { | ||
1041 | struct nvgpu_clk_arb *arb = | ||
1042 | container_of(work, struct nvgpu_clk_arb, update_fn_work); | ||
1043 | struct nvgpu_clk_session *session; | ||
1044 | struct nvgpu_clk_dev *dev; | ||
1045 | struct nvgpu_clk_dev *tmp; | ||
1046 | struct nvgpu_clk_arb_target *target, *actual; | ||
1047 | struct gk20a *g = arb->g; | ||
1048 | struct llist_node *head; | ||
1049 | |||
1050 | u32 pstate = VF_POINT_INVALID_PSTATE; | ||
1051 | u32 voltuv, voltuv_sram; | ||
1052 | bool mclk_set, gpc2clk_set; | ||
1053 | u32 nuvmin, nuvmin_sram; | ||
1054 | |||
1055 | u32 alarms_notified = 0; | ||
1056 | u32 current_alarm; | ||
1057 | int status = 0; | ||
1058 | |||
1059 | /* Temporary variables for checking target frequency */ | ||
1060 | u16 gpc2clk_target, sys2clk_target, xbar2clk_target, mclk_target; | ||
1061 | u16 gpc2clk_session_target, mclk_session_target; | ||
1062 | |||
1063 | #ifdef CONFIG_DEBUG_FS | ||
1064 | u64 t0, t1; | ||
1065 | struct nvgpu_clk_arb_debug *debug; | ||
1066 | |||
1067 | #endif | ||
1068 | |||
1069 | gk20a_dbg_fn(""); | ||
1070 | |||
1071 | /* bail out if gpu is down */ | ||
1072 | if (nvgpu_atomic64_read(&arb->alarm_mask) & EVENT(ALARM_GPU_LOST)) | ||
1073 | goto exit_arb; | ||
1074 | |||
1075 | #ifdef CONFIG_DEBUG_FS | ||
1076 | g->ops.bus.read_ptimer(g, &t0); | ||
1077 | #endif | ||
1078 | |||
1079 | /* Only one arbiter should be running */ | ||
1080 | gpc2clk_target = 0; | ||
1081 | mclk_target = 0; | ||
1082 | |||
1083 | rcu_read_lock(); | ||
1084 | list_for_each_entry_rcu(session, &arb->sessions, link) { | ||
1085 | if (!session->zombie) { | ||
1086 | mclk_set = false; | ||
1087 | gpc2clk_set = false; | ||
1088 | target = NV_ACCESS_ONCE(session->target) == | ||
1089 | &session->target_pool[0] ? | ||
1090 | &session->target_pool[1] : | ||
1091 | &session->target_pool[0]; | ||
1092 | /* Do not reorder pointer */ | ||
1093 | nvgpu_smp_rmb(); | ||
1094 | head = llist_del_all(&session->targets); | ||
1095 | if (head) { | ||
1096 | |||
1097 | /* Copy over state */ | ||
1098 | target->mclk = session->target->mclk; | ||
1099 | target->gpc2clk = session->target->gpc2clk; | ||
1100 | /* Query the latest committed request */ | ||
1101 | llist_for_each_entry_safe(dev, tmp, head, | ||
1102 | node) { | ||
1103 | if (!mclk_set && dev->mclk_target_mhz) { | ||
1104 | target->mclk = | ||
1105 | dev->mclk_target_mhz; | ||
1106 | mclk_set = true; | ||
1107 | } | ||
1108 | if (!gpc2clk_set && | ||
1109 | dev->gpc2clk_target_mhz) { | ||
1110 | target->gpc2clk = | ||
1111 | dev->gpc2clk_target_mhz; | ||
1112 | gpc2clk_set = true; | ||
1113 | } | ||
1114 | nvgpu_ref_get(&dev->refcount); | ||
1115 | llist_add(&dev->node, &arb->requests); | ||
1116 | } | ||
1117 | /* Ensure target is updated before ptr sawp */ | ||
1118 | nvgpu_smp_wmb(); | ||
1119 | xchg(&session->target, target); | ||
1120 | } | ||
1121 | |||
1122 | mclk_target = mclk_target > session->target->mclk ? | ||
1123 | mclk_target : session->target->mclk; | ||
1124 | |||
1125 | gpc2clk_target = | ||
1126 | gpc2clk_target > session->target->gpc2clk ? | ||
1127 | gpc2clk_target : session->target->gpc2clk; | ||
1128 | } | ||
1129 | } | ||
1130 | rcu_read_unlock(); | ||
1131 | |||
1132 | gpc2clk_target = (gpc2clk_target > 0) ? gpc2clk_target : | ||
1133 | arb->gpc2clk_default_mhz; | ||
1134 | |||
1135 | if (gpc2clk_target < arb->gpc2clk_min) | ||
1136 | gpc2clk_target = arb->gpc2clk_min; | ||
1137 | |||
1138 | if (gpc2clk_target > arb->gpc2clk_max) | ||
1139 | gpc2clk_target = arb->gpc2clk_max; | ||
1140 | |||
1141 | mclk_target = (mclk_target > 0) ? mclk_target: | ||
1142 | arb->mclk_default_mhz; | ||
1143 | |||
1144 | if (mclk_target < arb->mclk_min) | ||
1145 | mclk_target = arb->mclk_min; | ||
1146 | |||
1147 | if (mclk_target > arb->mclk_max) | ||
1148 | mclk_target = arb->mclk_max; | ||
1149 | |||
1150 | sys2clk_target = 0; | ||
1151 | xbar2clk_target = 0; | ||
1152 | |||
1153 | gpc2clk_session_target = gpc2clk_target; | ||
1154 | mclk_session_target = mclk_target; | ||
1155 | |||
1156 | /* Query the table for the closest vf point to program */ | ||
1157 | pstate = nvgpu_clk_arb_find_vf_point(arb, &gpc2clk_target, | ||
1158 | &sys2clk_target, &xbar2clk_target, &mclk_target, &voltuv, | ||
1159 | &voltuv_sram, &nuvmin, &nuvmin_sram); | ||
1160 | |||
1161 | if (pstate == VF_POINT_INVALID_PSTATE) { | ||
1162 | arb->status = -EINVAL; | ||
1163 | /* make status visible */ | ||
1164 | nvgpu_smp_mb(); | ||
1165 | goto exit_arb; | ||
1166 | } | ||
1167 | |||
1168 | if ((gpc2clk_target < gpc2clk_session_target) || | ||
1169 | (mclk_target < mclk_session_target)) | ||
1170 | nvgpu_clk_arb_set_global_alarm(g, | ||
1171 | EVENT(ALARM_TARGET_VF_NOT_POSSIBLE)); | ||
1172 | |||
1173 | if ((arb->actual->gpc2clk == gpc2clk_target) && | ||
1174 | (arb->actual->mclk == mclk_target) && | ||
1175 | (arb->voltuv_actual == voltuv)) { | ||
1176 | goto exit_arb; | ||
1177 | } | ||
1178 | |||
1179 | /* Program clocks */ | ||
1180 | /* A change in both mclk of gpc2clk may require a change in voltage */ | ||
1181 | |||
1182 | nvgpu_mutex_acquire(&arb->pstate_lock); | ||
1183 | status = nvgpu_lpwr_disable_pg(g, false); | ||
1184 | |||
1185 | status = clk_pmu_freq_controller_load(g, false, | ||
1186 | CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL); | ||
1187 | if (status < 0) { | ||
1188 | arb->status = status; | ||
1189 | nvgpu_mutex_release(&arb->pstate_lock); | ||
1190 | |||
1191 | /* make status visible */ | ||
1192 | nvgpu_smp_mb(); | ||
1193 | goto exit_arb; | ||
1194 | } | ||
1195 | status = volt_set_noiseaware_vmin(g, nuvmin, nuvmin_sram); | ||
1196 | if (status < 0) { | ||
1197 | arb->status = status; | ||
1198 | nvgpu_mutex_release(&arb->pstate_lock); | ||
1199 | |||
1200 | /* make status visible */ | ||
1201 | nvgpu_smp_mb(); | ||
1202 | goto exit_arb; | ||
1203 | } | ||
1204 | |||
1205 | status = nvgpu_clk_arb_change_vf_point(g, gpc2clk_target, | ||
1206 | sys2clk_target, xbar2clk_target, mclk_target, voltuv, | ||
1207 | voltuv_sram); | ||
1208 | if (status < 0) { | ||
1209 | arb->status = status; | ||
1210 | nvgpu_mutex_release(&arb->pstate_lock); | ||
1211 | |||
1212 | /* make status visible */ | ||
1213 | nvgpu_smp_mb(); | ||
1214 | goto exit_arb; | ||
1215 | } | ||
1216 | |||
1217 | status = clk_pmu_freq_controller_load(g, true, | ||
1218 | CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL); | ||
1219 | if (status < 0) { | ||
1220 | arb->status = status; | ||
1221 | nvgpu_mutex_release(&arb->pstate_lock); | ||
1222 | |||
1223 | /* make status visible */ | ||
1224 | nvgpu_smp_mb(); | ||
1225 | goto exit_arb; | ||
1226 | } | ||
1227 | |||
1228 | status = nvgpu_lwpr_mclk_change(g, pstate); | ||
1229 | if (status < 0) { | ||
1230 | arb->status = status; | ||
1231 | nvgpu_mutex_release(&arb->pstate_lock); | ||
1232 | |||
1233 | /* make status visible */ | ||
1234 | nvgpu_smp_mb(); | ||
1235 | goto exit_arb; | ||
1236 | } | ||
1237 | |||
1238 | actual = NV_ACCESS_ONCE(arb->actual) == &arb->actual_pool[0] ? | ||
1239 | &arb->actual_pool[1] : &arb->actual_pool[0]; | ||
1240 | |||
1241 | /* do not reorder this pointer */ | ||
1242 | nvgpu_smp_rmb(); | ||
1243 | actual->gpc2clk = gpc2clk_target; | ||
1244 | actual->mclk = mclk_target; | ||
1245 | arb->voltuv_actual = voltuv; | ||
1246 | actual->pstate = pstate; | ||
1247 | arb->status = status; | ||
1248 | |||
1249 | /* Make changes visible to other threads */ | ||
1250 | nvgpu_smp_wmb(); | ||
1251 | xchg(&arb->actual, actual); | ||
1252 | |||
1253 | status = nvgpu_lpwr_enable_pg(g, false); | ||
1254 | if (status < 0) { | ||
1255 | arb->status = status; | ||
1256 | nvgpu_mutex_release(&arb->pstate_lock); | ||
1257 | |||
1258 | /* make status visible */ | ||
1259 | nvgpu_smp_mb(); | ||
1260 | goto exit_arb; | ||
1261 | } | ||
1262 | |||
1263 | /* status must be visible before atomic inc */ | ||
1264 | nvgpu_smp_wmb(); | ||
1265 | nvgpu_atomic_inc(&arb->req_nr); | ||
1266 | |||
1267 | /* Unlock pstate change for PG */ | ||
1268 | nvgpu_mutex_release(&arb->pstate_lock); | ||
1269 | |||
1270 | /* VF Update complete */ | ||
1271 | nvgpu_clk_arb_set_global_alarm(g, EVENT(VF_UPDATE)); | ||
1272 | |||
1273 | nvgpu_cond_signal_interruptible(&arb->request_wq); | ||
1274 | |||
1275 | #ifdef CONFIG_DEBUG_FS | ||
1276 | g->ops.bus.read_ptimer(g, &t1); | ||
1277 | |||
1278 | debug = arb->debug == &arb->debug_pool[0] ? | ||
1279 | &arb->debug_pool[1] : &arb->debug_pool[0]; | ||
1280 | |||
1281 | memcpy(debug, arb->debug, sizeof(arb->debug_pool[0])); | ||
1282 | debug->switch_num++; | ||
1283 | |||
1284 | if (debug->switch_num == 1) { | ||
1285 | debug->switch_max = debug->switch_min = | ||
1286 | debug->switch_avg = (t1-t0)/1000; | ||
1287 | debug->switch_std = 0; | ||
1288 | } else { | ||
1289 | s64 prev_avg; | ||
1290 | s64 curr = (t1-t0)/1000; | ||
1291 | |||
1292 | debug->switch_max = curr > debug->switch_max ? | ||
1293 | curr : debug->switch_max; | ||
1294 | debug->switch_min = debug->switch_min ? | ||
1295 | (curr < debug->switch_min ? | ||
1296 | curr : debug->switch_min) : curr; | ||
1297 | prev_avg = debug->switch_avg; | ||
1298 | debug->switch_avg = (curr + | ||
1299 | (debug->switch_avg * (debug->switch_num-1))) / | ||
1300 | debug->switch_num; | ||
1301 | debug->switch_std += | ||
1302 | (curr - debug->switch_avg) * (curr - prev_avg); | ||
1303 | } | ||
1304 | /* commit changes before exchanging debug pointer */ | ||
1305 | nvgpu_smp_wmb(); | ||
1306 | xchg(&arb->debug, debug); | ||
1307 | #endif | ||
1308 | |||
1309 | exit_arb: | ||
1310 | if (status < 0) { | ||
1311 | nvgpu_err(g, "Error in arbiter update"); | ||
1312 | nvgpu_clk_arb_set_global_alarm(g, | ||
1313 | EVENT(ALARM_CLOCK_ARBITER_FAILED)); | ||
1314 | } | ||
1315 | |||
1316 | current_alarm = (u32) nvgpu_atomic64_read(&arb->alarm_mask); | ||
1317 | /* notify completion for all requests */ | ||
1318 | head = llist_del_all(&arb->requests); | ||
1319 | llist_for_each_entry_safe(dev, tmp, head, node) { | ||
1320 | nvgpu_atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM); | ||
1321 | nvgpu_cond_signal_interruptible(&dev->readout_wq); | ||
1322 | nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | ||
1323 | } | ||
1324 | |||
1325 | nvgpu_atomic_set(&arb->notification_queue.head, | ||
1326 | nvgpu_atomic_read(&arb->notification_queue.tail)); | ||
1327 | /* notify event for all users */ | ||
1328 | rcu_read_lock(); | ||
1329 | list_for_each_entry_rcu(dev, &arb->users, link) { | ||
1330 | alarms_notified |= | ||
1331 | nvgpu_clk_arb_notify(dev, arb->actual, current_alarm); | ||
1332 | } | ||
1333 | rcu_read_unlock(); | ||
1334 | |||
1335 | /* clear alarms */ | ||
1336 | nvgpu_clk_arb_clear_global_alarm(g, alarms_notified & | ||
1337 | ~EVENT(ALARM_GPU_LOST)); | ||
1338 | } | ||
1339 | |||
1340 | static void nvgpu_clk_arb_queue_notification(struct gk20a *g, | ||
1341 | struct nvgpu_clk_notification_queue *queue, | ||
1342 | u32 alarm_mask) { | ||
1343 | |||
1344 | u32 queue_index; | ||
1345 | u64 timestamp; | ||
1346 | |||
1347 | queue_index = (nvgpu_atomic_inc_return(&queue->tail)) % queue->size; | ||
1348 | /* get current timestamp */ | ||
1349 | timestamp = (u64) sched_clock(); | ||
1350 | |||
1351 | queue->notifications[queue_index].timestamp = timestamp; | ||
1352 | queue->notifications[queue_index].notification = alarm_mask; | ||
1353 | |||
1354 | } | ||
1355 | |||
1356 | static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, | ||
1357 | struct nvgpu_clk_arb_target *target, | ||
1358 | u32 alarm) { | ||
1359 | |||
1360 | struct nvgpu_clk_session *session = dev->session; | ||
1361 | struct nvgpu_clk_arb *arb = session->g->clk_arb; | ||
1362 | struct nvgpu_clk_notification *notification; | ||
1363 | |||
1364 | u32 queue_alarm_mask = 0; | ||
1365 | u32 enabled_mask = 0; | ||
1366 | u32 new_alarms_reported = 0; | ||
1367 | u32 poll_mask = 0; | ||
1368 | u32 tail, head; | ||
1369 | u32 queue_index; | ||
1370 | size_t size; | ||
1371 | int index; | ||
1372 | |||
1373 | enabled_mask = nvgpu_atomic_read(&dev->enabled_mask); | ||
1374 | size = arb->notification_queue.size; | ||
1375 | |||
1376 | /* queue global arbiter notifications in buffer */ | ||
1377 | do { | ||
1378 | tail = nvgpu_atomic_read(&arb->notification_queue.tail); | ||
1379 | /* copy items to the queue */ | ||
1380 | queue_index = nvgpu_atomic_read(&dev->queue.tail); | ||
1381 | head = dev->arb_queue_head; | ||
1382 | head = (tail - head) < arb->notification_queue.size ? | ||
1383 | head : tail - arb->notification_queue.size; | ||
1384 | |||
1385 | for (index = head; _WRAPGTEQ(tail, index); index++) { | ||
1386 | u32 alarm_detected; | ||
1387 | |||
1388 | notification = &arb->notification_queue. | ||
1389 | notifications[(index+1) % size]; | ||
1390 | alarm_detected = | ||
1391 | NV_ACCESS_ONCE(notification->notification); | ||
1392 | |||
1393 | if (!(enabled_mask & alarm_detected)) | ||
1394 | continue; | ||
1395 | |||
1396 | queue_index++; | ||
1397 | dev->queue.notifications[ | ||
1398 | queue_index % dev->queue.size].timestamp = | ||
1399 | NV_ACCESS_ONCE(notification->timestamp); | ||
1400 | |||
1401 | dev->queue.notifications[ | ||
1402 | queue_index % dev->queue.size].notification = | ||
1403 | alarm_detected; | ||
1404 | |||
1405 | queue_alarm_mask |= alarm_detected; | ||
1406 | } | ||
1407 | } while (unlikely(nvgpu_atomic_read(&arb->notification_queue.tail) != | ||
1408 | (int)tail)); | ||
1409 | |||
1410 | nvgpu_atomic_set(&dev->queue.tail, queue_index); | ||
1411 | /* update the last notification we processed from global queue */ | ||
1412 | |||
1413 | dev->arb_queue_head = tail; | ||
1414 | |||
1415 | /* Check if current session targets are met */ | ||
1416 | if (enabled_mask & EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)) { | ||
1417 | if ((target->gpc2clk < session->target->gpc2clk) | ||
1418 | || (target->mclk < session->target->mclk)) { | ||
1419 | |||
1420 | poll_mask |= (POLLIN | POLLPRI); | ||
1421 | nvgpu_clk_arb_queue_notification(arb->g, &dev->queue, | ||
1422 | EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)); | ||
1423 | } | ||
1424 | } | ||
1425 | |||
1426 | /* Check if there is a new VF update */ | ||
1427 | if (queue_alarm_mask & EVENT(VF_UPDATE)) | ||
1428 | poll_mask |= (POLLIN | POLLRDNORM); | ||
1429 | |||
1430 | /* Notify sticky alarms that were not reported on previous run*/ | ||
1431 | new_alarms_reported = (queue_alarm_mask | | ||
1432 | (alarm & ~dev->alarms_reported & queue_alarm_mask)); | ||
1433 | |||
1434 | if (new_alarms_reported & ~LOCAL_ALARM_MASK) { | ||
1435 | /* check that we are not re-reporting */ | ||
1436 | if (new_alarms_reported & EVENT(ALARM_GPU_LOST)) | ||
1437 | poll_mask |= POLLHUP; | ||
1438 | |||
1439 | poll_mask |= (POLLIN | POLLPRI); | ||
1440 | /* On next run do not report global alarms that were already | ||
1441 | * reported, but report SHUTDOWN always */ | ||
1442 | dev->alarms_reported = new_alarms_reported & ~LOCAL_ALARM_MASK & | ||
1443 | ~EVENT(ALARM_GPU_LOST); | ||
1444 | } | ||
1445 | |||
1446 | if (poll_mask) { | ||
1447 | nvgpu_atomic_set(&dev->poll_mask, poll_mask); | ||
1448 | nvgpu_cond_broadcast_interruptible(&dev->readout_wq); | ||
1449 | } | ||
1450 | |||
1451 | return new_alarms_reported; | ||
1452 | } | ||
1453 | |||
1454 | static int nvgpu_clk_arb_set_event_filter(struct nvgpu_clk_dev *dev, | ||
1455 | struct nvgpu_gpu_set_event_filter_args *args) | ||
1456 | { | ||
1457 | u32 mask; | ||
1458 | |||
1459 | gk20a_dbg(gpu_dbg_fn, ""); | ||
1460 | |||
1461 | if (args->flags) | ||
1462 | return -EINVAL; | ||
1463 | |||
1464 | if (args->size != 1) | ||
1465 | return -EINVAL; | ||
1466 | |||
1467 | if (copy_from_user(&mask, (void __user *) args->buffer, | ||
1468 | args->size * sizeof(u32))) | ||
1469 | return -EFAULT; | ||
1470 | |||
1471 | /* update alarm mask */ | ||
1472 | nvgpu_atomic_set(&dev->enabled_mask, mask); | ||
1473 | |||
1474 | return 0; | ||
1475 | } | ||
1476 | |||
1477 | static long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd, | ||
1478 | unsigned long arg) | ||
1479 | { | ||
1480 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1481 | struct gk20a *g = dev->session->g; | ||
1482 | u8 buf[NVGPU_EVENT_IOCTL_MAX_ARG_SIZE]; | ||
1483 | int err = 0; | ||
1484 | |||
1485 | gk20a_dbg(gpu_dbg_fn, "nr=%d", _IOC_NR(cmd)); | ||
1486 | |||
1487 | if ((_IOC_TYPE(cmd) != NVGPU_EVENT_IOCTL_MAGIC) || (_IOC_NR(cmd) == 0) | ||
1488 | || (_IOC_NR(cmd) > NVGPU_EVENT_IOCTL_LAST)) | ||
1489 | return -EINVAL; | ||
1490 | |||
1491 | BUG_ON(_IOC_SIZE(cmd) > NVGPU_EVENT_IOCTL_MAX_ARG_SIZE); | ||
1492 | |||
1493 | memset(buf, 0, sizeof(buf)); | ||
1494 | if (_IOC_DIR(cmd) & _IOC_WRITE) { | ||
1495 | if (copy_from_user(buf, (void __user *) arg, _IOC_SIZE(cmd))) | ||
1496 | return -EFAULT; | ||
1497 | } | ||
1498 | |||
1499 | switch (cmd) { | ||
1500 | case NVGPU_EVENT_IOCTL_SET_FILTER: | ||
1501 | err = nvgpu_clk_arb_set_event_filter(dev, | ||
1502 | (struct nvgpu_gpu_set_event_filter_args *)buf); | ||
1503 | break; | ||
1504 | default: | ||
1505 | nvgpu_warn(g, "unrecognized event ioctl cmd: 0x%x", cmd); | ||
1506 | err = -ENOTTY; | ||
1507 | } | ||
1508 | |||
1509 | if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ)) | ||
1510 | err = copy_to_user((void __user *) arg, buf, _IOC_SIZE(cmd)); | ||
1511 | |||
1512 | return err; | ||
1513 | } | ||
1514 | |||
1515 | int nvgpu_clk_arb_commit_request_fd(struct gk20a *g, | ||
1516 | struct nvgpu_clk_session *session, int request_fd) | ||
1517 | { | ||
1518 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1519 | struct nvgpu_clk_dev *dev; | ||
1520 | struct fd fd; | ||
1521 | int err = 0; | ||
1522 | |||
1523 | gk20a_dbg_fn(""); | ||
1524 | |||
1525 | fd = fdget(request_fd); | ||
1526 | if (!fd.file) | ||
1527 | return -EINVAL; | ||
1528 | |||
1529 | if (fd.file->f_op != &completion_dev_ops) { | ||
1530 | err = -EINVAL; | ||
1531 | goto fdput_fd; | ||
1532 | } | ||
1533 | |||
1534 | dev = (struct nvgpu_clk_dev *) fd.file->private_data; | ||
1535 | |||
1536 | if (!dev || dev->session != session) { | ||
1537 | err = -EINVAL; | ||
1538 | goto fdput_fd; | ||
1539 | } | ||
1540 | nvgpu_ref_get(&dev->refcount); | ||
1541 | llist_add(&dev->node, &session->targets); | ||
1542 | if (arb->update_work_queue) | ||
1543 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
1544 | |||
1545 | fdput_fd: | ||
1546 | fdput(fd); | ||
1547 | return err; | ||
1548 | } | ||
1549 | |||
1550 | static inline u32 __pending_event(struct nvgpu_clk_dev *dev, | ||
1551 | struct nvgpu_gpu_event_info *info) { | ||
1552 | |||
1553 | u32 tail, head; | ||
1554 | u32 events = 0; | ||
1555 | struct nvgpu_clk_notification *p_notif; | ||
1556 | |||
1557 | tail = nvgpu_atomic_read(&dev->queue.tail); | ||
1558 | head = nvgpu_atomic_read(&dev->queue.head); | ||
1559 | |||
1560 | head = (tail - head) < dev->queue.size ? head : tail - dev->queue.size; | ||
1561 | |||
1562 | if (_WRAPGTEQ(tail, head) && info) { | ||
1563 | head++; | ||
1564 | p_notif = &dev->queue.notifications[head % dev->queue.size]; | ||
1565 | events |= p_notif->notification; | ||
1566 | info->event_id = ffs(events) - 1; | ||
1567 | info->timestamp = p_notif->timestamp; | ||
1568 | nvgpu_atomic_set(&dev->queue.head, head); | ||
1569 | } | ||
1570 | |||
1571 | return events; | ||
1572 | } | ||
1573 | |||
1574 | static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf, | ||
1575 | size_t size, loff_t *off) | ||
1576 | { | ||
1577 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1578 | struct nvgpu_gpu_event_info info; | ||
1579 | ssize_t err; | ||
1580 | |||
1581 | gk20a_dbg_fn("filp=%p, buf=%p, size=%zu", filp, buf, size); | ||
1582 | |||
1583 | if ((size - *off) < sizeof(info)) | ||
1584 | return 0; | ||
1585 | |||
1586 | memset(&info, 0, sizeof(info)); | ||
1587 | /* Get the oldest event from the queue */ | ||
1588 | while (!__pending_event(dev, &info)) { | ||
1589 | if (filp->f_flags & O_NONBLOCK) | ||
1590 | return -EAGAIN; | ||
1591 | err = NVGPU_COND_WAIT_INTERRUPTIBLE(&dev->readout_wq, | ||
1592 | __pending_event(dev, &info), 0); | ||
1593 | if (err) | ||
1594 | return err; | ||
1595 | if (info.timestamp) | ||
1596 | break; | ||
1597 | } | ||
1598 | |||
1599 | if (copy_to_user(buf + *off, &info, sizeof(info))) | ||
1600 | return -EFAULT; | ||
1601 | |||
1602 | return sizeof(info); | ||
1603 | } | ||
1604 | |||
1605 | static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait) | ||
1606 | { | ||
1607 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1608 | |||
1609 | gk20a_dbg_fn(""); | ||
1610 | |||
1611 | poll_wait(filp, &dev->readout_wq.wq, wait); | ||
1612 | return nvgpu_atomic_xchg(&dev->poll_mask, 0); | ||
1613 | } | ||
1614 | |||
1615 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, | ||
1616 | struct file *filp) | ||
1617 | { | ||
1618 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1619 | struct nvgpu_clk_session *session = dev->session; | ||
1620 | |||
1621 | |||
1622 | gk20a_dbg_fn(""); | ||
1623 | |||
1624 | nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session); | ||
1625 | nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | ||
1626 | return 0; | ||
1627 | } | ||
1628 | |||
1629 | static int nvgpu_clk_arb_release_event_dev(struct inode *inode, | ||
1630 | struct file *filp) | ||
1631 | { | ||
1632 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1633 | struct nvgpu_clk_session *session = dev->session; | ||
1634 | struct nvgpu_clk_arb *arb; | ||
1635 | |||
1636 | arb = session->g->clk_arb; | ||
1637 | |||
1638 | gk20a_dbg_fn(""); | ||
1639 | |||
1640 | if (arb) { | ||
1641 | nvgpu_spinlock_acquire(&arb->users_lock); | ||
1642 | list_del_rcu(&dev->link); | ||
1643 | nvgpu_spinlock_release(&arb->users_lock); | ||
1644 | nvgpu_clk_notification_queue_free(arb->g, &dev->queue); | ||
1645 | } | ||
1646 | |||
1647 | synchronize_rcu(); | ||
1648 | nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session); | ||
1649 | nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | ||
1650 | |||
1651 | return 0; | ||
1652 | } | ||
1653 | |||
1654 | int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session, | ||
1655 | int request_fd, u32 api_domain, u16 target_mhz) | ||
1656 | { | ||
1657 | struct nvgpu_clk_dev *dev; | ||
1658 | struct fd fd; | ||
1659 | int err = 0; | ||
1660 | |||
1661 | gk20a_dbg_fn("domain=0x%08x target_mhz=%u", api_domain, target_mhz); | ||
1662 | |||
1663 | fd = fdget(request_fd); | ||
1664 | if (!fd.file) | ||
1665 | return -EINVAL; | ||
1666 | |||
1667 | if (fd.file->f_op != &completion_dev_ops) { | ||
1668 | err = -EINVAL; | ||
1669 | goto fdput_fd; | ||
1670 | } | ||
1671 | |||
1672 | dev = fd.file->private_data; | ||
1673 | if (!dev || dev->session != session) { | ||
1674 | err = -EINVAL; | ||
1675 | goto fdput_fd; | ||
1676 | } | ||
1677 | |||
1678 | switch (api_domain) { | ||
1679 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1680 | dev->mclk_target_mhz = target_mhz; | ||
1681 | break; | ||
1682 | |||
1683 | case NVGPU_GPU_CLK_DOMAIN_GPCCLK: | ||
1684 | dev->gpc2clk_target_mhz = target_mhz * 2ULL; | ||
1685 | break; | ||
1686 | |||
1687 | default: | ||
1688 | err = -EINVAL; | ||
1689 | } | ||
1690 | |||
1691 | fdput_fd: | ||
1692 | fdput(fd); | ||
1693 | return err; | ||
1694 | } | ||
1695 | |||
1696 | int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session, | ||
1697 | u32 api_domain, u16 *freq_mhz) | ||
1698 | { | ||
1699 | int err = 0; | ||
1700 | struct nvgpu_clk_arb_target *target; | ||
1701 | |||
1702 | do { | ||
1703 | target = NV_ACCESS_ONCE(session->target); | ||
1704 | /* no reordering of this pointer */ | ||
1705 | nvgpu_smp_rmb(); | ||
1706 | |||
1707 | switch (api_domain) { | ||
1708 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1709 | *freq_mhz = target->mclk; | ||
1710 | break; | ||
1711 | |||
1712 | case NVGPU_GPU_CLK_DOMAIN_GPCCLK: | ||
1713 | *freq_mhz = target->gpc2clk / 2ULL; | ||
1714 | break; | ||
1715 | |||
1716 | default: | ||
1717 | *freq_mhz = 0; | ||
1718 | err = -EINVAL; | ||
1719 | } | ||
1720 | } while (target != NV_ACCESS_ONCE(session->target)); | ||
1721 | return err; | ||
1722 | } | ||
1723 | |||
1724 | int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g, | ||
1725 | u32 api_domain, u16 *freq_mhz) | ||
1726 | { | ||
1727 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1728 | int err = 0; | ||
1729 | struct nvgpu_clk_arb_target *actual; | ||
1730 | |||
1731 | do { | ||
1732 | actual = NV_ACCESS_ONCE(arb->actual); | ||
1733 | /* no reordering of this pointer */ | ||
1734 | nvgpu_smp_rmb(); | ||
1735 | |||
1736 | switch (api_domain) { | ||
1737 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1738 | *freq_mhz = actual->mclk; | ||
1739 | break; | ||
1740 | |||
1741 | case NVGPU_GPU_CLK_DOMAIN_GPCCLK: | ||
1742 | *freq_mhz = actual->gpc2clk / 2ULL; | ||
1743 | break; | ||
1744 | |||
1745 | default: | ||
1746 | *freq_mhz = 0; | ||
1747 | err = -EINVAL; | ||
1748 | } | ||
1749 | } while (actual != NV_ACCESS_ONCE(arb->actual)); | ||
1750 | return err; | ||
1751 | } | ||
1752 | |||
1753 | int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g, | ||
1754 | u32 api_domain, u16 *freq_mhz) | ||
1755 | { | ||
1756 | switch(api_domain) { | ||
1757 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1758 | *freq_mhz = g->ops.clk.measure_freq(g, CTRL_CLK_DOMAIN_MCLK) / | ||
1759 | 1000000ULL; | ||
1760 | return 0; | ||
1761 | |||
1762 | case NVGPU_GPU_CLK_DOMAIN_GPCCLK: | ||
1763 | *freq_mhz = g->ops.clk.measure_freq(g, CTRL_CLK_DOMAIN_GPC2CLK) / | ||
1764 | 2000000ULL; | ||
1765 | return 0; | ||
1766 | |||
1767 | default: | ||
1768 | return -EINVAL; | ||
1769 | } | ||
1770 | } | ||
1771 | |||
1772 | int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain, | ||
1773 | u16 *min_mhz, u16 *max_mhz) | ||
1774 | { | ||
1775 | int ret; | ||
1776 | |||
1777 | switch(api_domain) { | ||
1778 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1779 | ret = g->ops.clk_arb.get_arbiter_clk_range(g, | ||
1780 | CTRL_CLK_DOMAIN_MCLK, min_mhz, max_mhz); | ||
1781 | return ret; | ||
1782 | |||
1783 | case NVGPU_GPU_CLK_DOMAIN_GPCCLK: | ||
1784 | ret = g->ops.clk_arb.get_arbiter_clk_range(g, | ||
1785 | CTRL_CLK_DOMAIN_GPC2CLK, min_mhz, max_mhz); | ||
1786 | if (!ret) { | ||
1787 | *min_mhz /= 2; | ||
1788 | *max_mhz /= 2; | ||
1789 | } | ||
1790 | return ret; | ||
1791 | |||
1792 | default: | ||
1793 | return -EINVAL; | ||
1794 | } | ||
1795 | } | ||
1796 | |||
1797 | u32 nvgpu_clk_arb_get_arbiter_clk_domains(struct gk20a *g) | ||
1798 | { | ||
1799 | u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g); | ||
1800 | u32 api_domains = 0; | ||
1801 | |||
1802 | if (clk_domains & CTRL_CLK_DOMAIN_GPC2CLK) | ||
1803 | api_domains |= BIT(NVGPU_GPU_CLK_DOMAIN_GPCCLK); | ||
1804 | |||
1805 | if (clk_domains & CTRL_CLK_DOMAIN_MCLK) | ||
1806 | api_domains |= BIT(NVGPU_GPU_CLK_DOMAIN_MCLK); | ||
1807 | |||
1808 | return api_domains; | ||
1809 | } | ||
1810 | |||
1811 | bool nvgpu_clk_arb_is_valid_domain(struct gk20a *g, u32 api_domain) | ||
1812 | { | ||
1813 | u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g); | ||
1814 | |||
1815 | switch(api_domain) { | ||
1816 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1817 | return ((clk_domains & CTRL_CLK_DOMAIN_MCLK) != 0); | ||
1818 | |||
1819 | case NVGPU_GPU_CLK_DOMAIN_GPCCLK: | ||
1820 | return ((clk_domains & CTRL_CLK_DOMAIN_GPC2CLK) != 0); | ||
1821 | |||
1822 | default: | ||
1823 | return false; | ||
1824 | } | ||
1825 | } | ||
1826 | |||
1827 | int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g, | ||
1828 | u32 api_domain, u32 *max_points, u16 *fpoints) | ||
1829 | { | ||
1830 | int err; | ||
1831 | u32 i; | ||
1832 | |||
1833 | switch (api_domain) { | ||
1834 | case NVGPU_GPU_CLK_DOMAIN_GPCCLK: | ||
1835 | err = clk_domain_get_f_points(g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
1836 | max_points, fpoints); | ||
1837 | if (err || !fpoints) | ||
1838 | return err; | ||
1839 | for (i = 0; i < *max_points; i++) | ||
1840 | fpoints[i] /= 2; | ||
1841 | return 0; | ||
1842 | case NVGPU_GPU_CLK_DOMAIN_MCLK: | ||
1843 | return clk_domain_get_f_points(g, CTRL_CLK_DOMAIN_MCLK, | ||
1844 | max_points, fpoints); | ||
1845 | default: | ||
1846 | return -EINVAL; | ||
1847 | } | ||
1848 | } | ||
1849 | |||
1850 | static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb, | ||
1851 | u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk, | ||
1852 | u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram) | ||
1853 | { | ||
1854 | u16 gpc2clk_target, mclk_target; | ||
1855 | u32 gpc2clk_voltuv, gpc2clk_voltuv_sram; | ||
1856 | u32 mclk_voltuv, mclk_voltuv_sram; | ||
1857 | u32 pstate = VF_POINT_INVALID_PSTATE; | ||
1858 | struct nvgpu_clk_vf_table *table; | ||
1859 | u32 index, index_mclk; | ||
1860 | struct nvgpu_clk_vf_point *mclk_vf = NULL; | ||
1861 | |||
1862 | do { | ||
1863 | gpc2clk_target = *gpc2clk; | ||
1864 | mclk_target = *mclk; | ||
1865 | gpc2clk_voltuv = 0; | ||
1866 | gpc2clk_voltuv_sram = 0; | ||
1867 | mclk_voltuv = 0; | ||
1868 | mclk_voltuv_sram = 0; | ||
1869 | |||
1870 | table = NV_ACCESS_ONCE(arb->current_vf_table); | ||
1871 | /* pointer to table can be updated by callback */ | ||
1872 | nvgpu_smp_rmb(); | ||
1873 | |||
1874 | if (!table) | ||
1875 | continue; | ||
1876 | if ((!table->gpc2clk_num_points) || (!table->mclk_num_points)) { | ||
1877 | nvgpu_err(arb->g, "found empty table"); | ||
1878 | goto find_exit; | ||
1879 | } | ||
1880 | /* First we check MCLK to find out which PSTATE we are | ||
1881 | * are requesting, and from there try to find the minimum | ||
1882 | * GPC2CLK on the same PSTATE that satisfies the request. | ||
1883 | * If no GPC2CLK can be found, then we need to up the PSTATE | ||
1884 | */ | ||
1885 | |||
1886 | recalculate_vf_point: | ||
1887 | for (index = 0; index < table->mclk_num_points; index++) { | ||
1888 | if (table->mclk_points[index].mem_mhz >= mclk_target) { | ||
1889 | mclk_vf = &table->mclk_points[index]; | ||
1890 | break; | ||
1891 | } | ||
1892 | } | ||
1893 | if (index == table->mclk_num_points) { | ||
1894 | mclk_vf = &table->mclk_points[index-1]; | ||
1895 | index = table->mclk_num_points - 1; | ||
1896 | } | ||
1897 | index_mclk = index; | ||
1898 | |||
1899 | /* round up the freq requests */ | ||
1900 | for (index = 0; index < table->gpc2clk_num_points; index++) { | ||
1901 | pstate = VF_POINT_COMMON_PSTATE( | ||
1902 | &table->gpc2clk_points[index], mclk_vf); | ||
1903 | |||
1904 | if ((table->gpc2clk_points[index].gpc_mhz >= | ||
1905 | gpc2clk_target) && | ||
1906 | (pstate != VF_POINT_INVALID_PSTATE)){ | ||
1907 | gpc2clk_target = | ||
1908 | table->gpc2clk_points[index].gpc_mhz; | ||
1909 | *sys2clk = | ||
1910 | table->gpc2clk_points[index].sys_mhz; | ||
1911 | *xbar2clk = | ||
1912 | table->gpc2clk_points[index].xbar_mhz; | ||
1913 | |||
1914 | gpc2clk_voltuv = | ||
1915 | table->gpc2clk_points[index].uvolt; | ||
1916 | gpc2clk_voltuv_sram = | ||
1917 | table->gpc2clk_points[index].uvolt_sram; | ||
1918 | break; | ||
1919 | } | ||
1920 | } | ||
1921 | |||
1922 | if (index == table->gpc2clk_num_points) { | ||
1923 | pstate = VF_POINT_COMMON_PSTATE( | ||
1924 | &table->gpc2clk_points[index-1], mclk_vf); | ||
1925 | if (pstate != VF_POINT_INVALID_PSTATE) { | ||
1926 | gpc2clk_target = | ||
1927 | table->gpc2clk_points[index-1].gpc_mhz; | ||
1928 | *sys2clk = | ||
1929 | table->gpc2clk_points[index-1].sys_mhz; | ||
1930 | *xbar2clk = | ||
1931 | table->gpc2clk_points[index-1].xbar_mhz; | ||
1932 | |||
1933 | gpc2clk_voltuv = | ||
1934 | table->gpc2clk_points[index-1].uvolt; | ||
1935 | gpc2clk_voltuv_sram = | ||
1936 | table->gpc2clk_points[index-1]. | ||
1937 | uvolt_sram; | ||
1938 | } else if (index_mclk >= table->mclk_num_points - 1) { | ||
1939 | /* There is no available combination of MCLK | ||
1940 | * and GPC2CLK, we need to fail this | ||
1941 | */ | ||
1942 | gpc2clk_target = 0; | ||
1943 | mclk_target = 0; | ||
1944 | pstate = VF_POINT_INVALID_PSTATE; | ||
1945 | goto find_exit; | ||
1946 | } else { | ||
1947 | /* recalculate with higher PSTATE */ | ||
1948 | gpc2clk_target = *gpc2clk; | ||
1949 | mclk_target = table->mclk_points[index_mclk+1]. | ||
1950 | mem_mhz; | ||
1951 | goto recalculate_vf_point; | ||
1952 | } | ||
1953 | } | ||
1954 | |||
1955 | mclk_target = mclk_vf->mem_mhz; | ||
1956 | mclk_voltuv = mclk_vf->uvolt; | ||
1957 | mclk_voltuv_sram = mclk_vf->uvolt_sram; | ||
1958 | |||
1959 | } while (!table || | ||
1960 | (NV_ACCESS_ONCE(arb->current_vf_table) != table)); | ||
1961 | |||
1962 | find_exit: | ||
1963 | *voltuv = gpc2clk_voltuv > mclk_voltuv ? gpc2clk_voltuv : mclk_voltuv; | ||
1964 | *voltuv_sram = gpc2clk_voltuv_sram > mclk_voltuv_sram ? | ||
1965 | gpc2clk_voltuv_sram : mclk_voltuv_sram; | ||
1966 | /* noise unaware vmin */ | ||
1967 | *nuvmin = mclk_voltuv; | ||
1968 | *nuvmin_sram = mclk_voltuv_sram; | ||
1969 | *gpc2clk = gpc2clk_target < *gpc2clk ? gpc2clk_target : *gpc2clk; | ||
1970 | *mclk = mclk_target; | ||
1971 | return pstate; | ||
1972 | } | ||
1973 | |||
1974 | /* This function is inherently unsafe to call while arbiter is running | ||
1975 | * arbiter must be blocked before calling this function */ | ||
1976 | int nvgpu_clk_arb_get_current_pstate(struct gk20a *g) | ||
1977 | { | ||
1978 | return NV_ACCESS_ONCE(g->clk_arb->actual->pstate); | ||
1979 | } | ||
1980 | |||
1981 | static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target, | ||
1982 | u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv, | ||
1983 | u32 voltuv_sram) | ||
1984 | { | ||
1985 | struct set_fll_clk fllclk; | ||
1986 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1987 | int status; | ||
1988 | |||
1989 | fllclk.gpc2clkmhz = gpc2clk_target; | ||
1990 | fllclk.sys2clkmhz = sys2clk_target; | ||
1991 | fllclk.xbar2clkmhz = xbar2clk_target; | ||
1992 | |||
1993 | fllclk.voltuv = voltuv; | ||
1994 | |||
1995 | /* if voltage ascends we do: | ||
1996 | * (1) FLL change | ||
1997 | * (2) Voltage change | ||
1998 | * (3) MCLK change | ||
1999 | * If it goes down | ||
2000 | * (1) MCLK change | ||
2001 | * (2) Voltage change | ||
2002 | * (3) FLL change | ||
2003 | */ | ||
2004 | |||
2005 | /* descending */ | ||
2006 | if (voltuv < arb->voltuv_actual) { | ||
2007 | status = g->ops.clk.mclk_change(g, mclk_target); | ||
2008 | if (status < 0) | ||
2009 | return status; | ||
2010 | |||
2011 | status = volt_set_voltage(g, voltuv, voltuv_sram); | ||
2012 | if (status < 0) | ||
2013 | return status; | ||
2014 | |||
2015 | status = clk_set_fll_clks(g, &fllclk); | ||
2016 | if (status < 0) | ||
2017 | return status; | ||
2018 | } else { | ||
2019 | status = clk_set_fll_clks(g, &fllclk); | ||
2020 | if (status < 0) | ||
2021 | return status; | ||
2022 | |||
2023 | status = volt_set_voltage(g, voltuv, voltuv_sram); | ||
2024 | if (status < 0) | ||
2025 | return status; | ||
2026 | |||
2027 | status = g->ops.clk.mclk_change(g, mclk_target); | ||
2028 | if (status < 0) | ||
2029 | return status; | ||
2030 | } | ||
2031 | |||
2032 | return 0; | ||
2033 | } | ||
2034 | |||
2035 | void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock) | ||
2036 | { | ||
2037 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
2038 | |||
2039 | if (lock) | ||
2040 | nvgpu_mutex_acquire(&arb->pstate_lock); | ||
2041 | else | ||
2042 | nvgpu_mutex_release(&arb->pstate_lock); | ||
2043 | } | ||
2044 | |||
2045 | #ifdef CONFIG_DEBUG_FS | ||
2046 | static int nvgpu_clk_arb_stats_show(struct seq_file *s, void *unused) | ||
2047 | { | ||
2048 | struct gk20a *g = s->private; | ||
2049 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
2050 | struct nvgpu_clk_arb_debug *debug; | ||
2051 | |||
2052 | u64 num; | ||
2053 | s64 tmp, avg, std, max, min; | ||
2054 | |||
2055 | debug = NV_ACCESS_ONCE(arb->debug); | ||
2056 | /* Make copy of structure and ensure no reordering */ | ||
2057 | nvgpu_smp_rmb(); | ||
2058 | if (!debug) | ||
2059 | return -EINVAL; | ||
2060 | |||
2061 | std = debug->switch_std; | ||
2062 | avg = debug->switch_avg; | ||
2063 | max = debug->switch_max; | ||
2064 | min = debug->switch_min; | ||
2065 | num = debug->switch_num; | ||
2066 | |||
2067 | tmp = std; | ||
2068 | do_div(tmp, num); | ||
2069 | seq_printf(s, "Number of transitions: %lld\n", | ||
2070 | num); | ||
2071 | seq_printf(s, "max / min : %lld / %lld usec\n", | ||
2072 | max, min); | ||
2073 | seq_printf(s, "avg / std : %lld / %ld usec\n", | ||
2074 | avg, int_sqrt(tmp)); | ||
2075 | |||
2076 | return 0; | ||
2077 | } | ||
2078 | |||
2079 | static int nvgpu_clk_arb_stats_open(struct inode *inode, struct file *file) | ||
2080 | { | ||
2081 | return single_open(file, nvgpu_clk_arb_stats_show, inode->i_private); | ||
2082 | } | ||
2083 | |||
2084 | static const struct file_operations nvgpu_clk_arb_stats_fops = { | ||
2085 | .open = nvgpu_clk_arb_stats_open, | ||
2086 | .read = seq_read, | ||
2087 | .llseek = seq_lseek, | ||
2088 | .release = single_release, | ||
2089 | }; | ||
2090 | |||
2091 | |||
2092 | static int nvgpu_clk_arb_debugfs_init(struct gk20a *g) | ||
2093 | { | ||
2094 | struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); | ||
2095 | struct dentry *gpu_root = l->debugfs; | ||
2096 | struct dentry *d; | ||
2097 | |||
2098 | gk20a_dbg(gpu_dbg_info, "g=%p", g); | ||
2099 | |||
2100 | d = debugfs_create_file( | ||
2101 | "arb_stats", | ||
2102 | S_IRUGO, | ||
2103 | gpu_root, | ||
2104 | g, | ||
2105 | &nvgpu_clk_arb_stats_fops); | ||
2106 | if (!d) | ||
2107 | return -ENOMEM; | ||
2108 | |||
2109 | return 0; | ||
2110 | } | ||
2111 | #endif | ||