diff options
-rw-r--r-- | drivers/gpu/nvgpu/clk/clk_arb.c | 516 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/clk/clk_arb.h | 3 |
2 files changed, 408 insertions, 111 deletions
diff --git a/drivers/gpu/nvgpu/clk/clk_arb.c b/drivers/gpu/nvgpu/clk/clk_arb.c index 673f3721..b4ed373d 100644 --- a/drivers/gpu/nvgpu/clk/clk_arb.c +++ b/drivers/gpu/nvgpu/clk/clk_arb.c | |||
@@ -25,6 +25,11 @@ | |||
25 | 25 | ||
26 | 26 | ||
27 | #define MAX_F_POINTS 127 | 27 | #define MAX_F_POINTS 127 |
28 | #define DEFAULT_EVENT_NUMBER 32 | ||
29 | |||
30 | struct nvgpu_clk_dev; | ||
31 | struct nvgpu_clk_arb_target; | ||
32 | struct nvgpu_clk_notification_queue; | ||
28 | 33 | ||
29 | #ifdef CONFIG_DEBUG_FS | 34 | #ifdef CONFIG_DEBUG_FS |
30 | static int nvgpu_clk_arb_debugfs_init(struct gk20a *g); | 35 | static int nvgpu_clk_arb_debugfs_init(struct gk20a *g); |
@@ -34,12 +39,12 @@ static int nvgpu_clk_arb_release_event_dev(struct inode *inode, | |||
34 | struct file *filp); | 39 | struct file *filp); |
35 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, | 40 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, |
36 | struct file *filp); | 41 | struct file *filp); |
37 | static unsigned int nvgpu_clk_arb_poll_completion_dev(struct file *filp, poll_table *wait); | 42 | static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait); |
38 | static unsigned int nvgpu_clk_arb_poll_event_dev(struct file *filp, poll_table *wait); | 43 | static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf, |
39 | static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, | 44 | size_t size, loff_t *off); |
40 | char __user *buf, size_t size, loff_t *off); | 45 | |
41 | long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd, | 46 | static long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd, |
42 | unsigned long arg); | 47 | unsigned long arg); |
43 | 48 | ||
44 | static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work); | 49 | static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work); |
45 | static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work); | 50 | static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work); |
@@ -52,6 +57,21 @@ static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target, | |||
52 | static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb, | 57 | static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb, |
53 | u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk, | 58 | u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk, |
54 | u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram); | 59 | u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram); |
60 | static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, | ||
61 | struct nvgpu_clk_arb_target *target, | ||
62 | u32 alarm_mask); | ||
63 | static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm); | ||
64 | static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm); | ||
65 | |||
66 | static void nvgpu_clk_arb_queue_notification(struct gk20a *g, | ||
67 | struct nvgpu_clk_notification_queue *queue, | ||
68 | u32 alarm_mask); | ||
69 | static int nvgpu_clk_notification_queue_alloc( | ||
70 | struct nvgpu_clk_notification_queue *queue, | ||
71 | size_t events_number); | ||
72 | |||
73 | static void nvgpu_clk_notification_queue_free( | ||
74 | struct nvgpu_clk_notification_queue *queue); | ||
55 | 75 | ||
56 | #define VF_POINT_INVALID_PSTATE ~0U | 76 | #define VF_POINT_INVALID_PSTATE ~0U |
57 | #define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b))) | 77 | #define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b))) |
@@ -62,6 +82,26 @@ static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb, | |||
62 | __fls((a)->pstates & (b)->pstates) :\ | 82 | __fls((a)->pstates & (b)->pstates) :\ |
63 | VF_POINT_INVALID_PSTATE) | 83 | VF_POINT_INVALID_PSTATE) |
64 | 84 | ||
85 | /* Local Alarms */ | ||
86 | #define EVENT(alarm) (0x1UL << NVGPU_GPU_EVENT_##alarm) | ||
87 | |||
88 | #define LOCAL_ALARM_MASK (EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE) | \ | ||
89 | EVENT(VF_UPDATE)) | ||
90 | |||
91 | #define _WRAPGTEQ(a, b) ((a-b) > 0) | ||
92 | |||
93 | struct nvgpu_clk_notification { | ||
94 | u32 notification; | ||
95 | u64 timestamp; | ||
96 | }; | ||
97 | |||
98 | struct nvgpu_clk_notification_queue { | ||
99 | u32 size; | ||
100 | atomic_t head; | ||
101 | atomic_t tail; | ||
102 | struct nvgpu_clk_notification *notifications; | ||
103 | }; | ||
104 | |||
65 | struct nvgpu_clk_vf_point { | 105 | struct nvgpu_clk_vf_point { |
66 | u16 pstates; | 106 | u16 pstates; |
67 | union { | 107 | union { |
@@ -135,6 +175,9 @@ struct nvgpu_clk_arb { | |||
135 | u16 *gpc2clk_f_points; | 175 | u16 *gpc2clk_f_points; |
136 | u32 gpc2clk_f_numpoints; | 176 | u32 gpc2clk_f_numpoints; |
137 | 177 | ||
178 | atomic64_t alarm_mask; | ||
179 | struct nvgpu_clk_notification_queue notification_queue; | ||
180 | |||
138 | #ifdef CONFIG_DEBUG_FS | 181 | #ifdef CONFIG_DEBUG_FS |
139 | struct nvgpu_clk_arb_debug debug_pool[2]; | 182 | struct nvgpu_clk_arb_debug debug_pool[2]; |
140 | struct nvgpu_clk_arb_debug *debug; | 183 | struct nvgpu_clk_arb_debug *debug; |
@@ -152,9 +195,10 @@ struct nvgpu_clk_dev { | |||
152 | atomic_t poll_mask; | 195 | atomic_t poll_mask; |
153 | u16 gpc2clk_target_mhz; | 196 | u16 gpc2clk_target_mhz; |
154 | u16 mclk_target_mhz; | 197 | u16 mclk_target_mhz; |
155 | spinlock_t event_lock; | 198 | u32 alarms_reported; |
156 | u32 event_status; | 199 | atomic_t enabled_mask; |
157 | u32 event_mask; | 200 | struct nvgpu_clk_notification_queue queue; |
201 | u32 arb_queue_head; | ||
158 | struct kref refcount; | 202 | struct kref refcount; |
159 | }; | 203 | }; |
160 | 204 | ||
@@ -172,13 +216,13 @@ struct nvgpu_clk_session { | |||
172 | static const struct file_operations completion_dev_ops = { | 216 | static const struct file_operations completion_dev_ops = { |
173 | .owner = THIS_MODULE, | 217 | .owner = THIS_MODULE, |
174 | .release = nvgpu_clk_arb_release_completion_dev, | 218 | .release = nvgpu_clk_arb_release_completion_dev, |
175 | .poll = nvgpu_clk_arb_poll_completion_dev, | 219 | .poll = nvgpu_clk_arb_poll_dev, |
176 | }; | 220 | }; |
177 | 221 | ||
178 | static const struct file_operations event_dev_ops = { | 222 | static const struct file_operations event_dev_ops = { |
179 | .owner = THIS_MODULE, | 223 | .owner = THIS_MODULE, |
180 | .release = nvgpu_clk_arb_release_event_dev, | 224 | .release = nvgpu_clk_arb_release_event_dev, |
181 | .poll = nvgpu_clk_arb_poll_event_dev, | 225 | .poll = nvgpu_clk_arb_poll_dev, |
182 | .read = nvgpu_clk_arb_read_event_dev, | 226 | .read = nvgpu_clk_arb_read_event_dev, |
183 | #ifdef CONFIG_COMPAT | 227 | #ifdef CONFIG_COMPAT |
184 | .compat_ioctl = nvgpu_clk_arb_ioctl_event_dev, | 228 | .compat_ioctl = nvgpu_clk_arb_ioctl_event_dev, |
@@ -186,6 +230,29 @@ static const struct file_operations event_dev_ops = { | |||
186 | .unlocked_ioctl = nvgpu_clk_arb_ioctl_event_dev, | 230 | .unlocked_ioctl = nvgpu_clk_arb_ioctl_event_dev, |
187 | }; | 231 | }; |
188 | 232 | ||
233 | static int nvgpu_clk_notification_queue_alloc( | ||
234 | struct nvgpu_clk_notification_queue *queue, | ||
235 | size_t events_number) { | ||
236 | queue->notifications = kcalloc(events_number, | ||
237 | sizeof(struct nvgpu_clk_notification), GFP_KERNEL); | ||
238 | if (!queue->notifications) | ||
239 | return -ENOMEM; | ||
240 | queue->size = events_number; | ||
241 | |||
242 | atomic_set(&queue->head, 0); | ||
243 | atomic_set(&queue->tail, 0); | ||
244 | |||
245 | return 0; | ||
246 | } | ||
247 | |||
248 | static void nvgpu_clk_notification_queue_free( | ||
249 | struct nvgpu_clk_notification_queue *queue) { | ||
250 | kfree(queue->notifications); | ||
251 | queue->size = 0; | ||
252 | atomic_set(&queue->head, 0); | ||
253 | atomic_set(&queue->tail, 0); | ||
254 | } | ||
255 | |||
189 | int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | 256 | int nvgpu_clk_arb_init_arbiter(struct gk20a *g) |
190 | { | 257 | { |
191 | struct nvgpu_clk_arb *arb; | 258 | struct nvgpu_clk_arb *arb; |
@@ -247,7 +314,7 @@ int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | |||
247 | 314 | ||
248 | err = g->ops.clk_arb.get_arbiter_clk_default(g, | 315 | err = g->ops.clk_arb.get_arbiter_clk_default(g, |
249 | NVGPU_GPU_CLK_DOMAIN_MCLK, &default_mhz); | 316 | NVGPU_GPU_CLK_DOMAIN_MCLK, &default_mhz); |
250 | if (err) { | 317 | if (err < 0) { |
251 | err = -EINVAL; | 318 | err = -EINVAL; |
252 | goto init_fail; | 319 | goto init_fail; |
253 | } | 320 | } |
@@ -256,7 +323,7 @@ int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | |||
256 | 323 | ||
257 | err = g->ops.clk_arb.get_arbiter_clk_default(g, | 324 | err = g->ops.clk_arb.get_arbiter_clk_default(g, |
258 | NVGPU_GPU_CLK_DOMAIN_GPC2CLK, &default_mhz); | 325 | NVGPU_GPU_CLK_DOMAIN_GPC2CLK, &default_mhz); |
259 | if (err) { | 326 | if (err < 0) { |
260 | err = -EINVAL; | 327 | err = -EINVAL; |
261 | goto init_fail; | 328 | goto init_fail; |
262 | } | 329 | } |
@@ -267,6 +334,12 @@ int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | |||
267 | 334 | ||
268 | atomic_set(&arb->req_nr, 0); | 335 | atomic_set(&arb->req_nr, 0); |
269 | 336 | ||
337 | atomic64_set(&arb->alarm_mask, 0); | ||
338 | err = nvgpu_clk_notification_queue_alloc(&arb->notification_queue, | ||
339 | DEFAULT_EVENT_NUMBER); | ||
340 | if (err < 0) | ||
341 | goto init_fail; | ||
342 | |||
270 | INIT_LIST_HEAD_RCU(&arb->users); | 343 | INIT_LIST_HEAD_RCU(&arb->users); |
271 | INIT_LIST_HEAD_RCU(&arb->sessions); | 344 | INIT_LIST_HEAD_RCU(&arb->sessions); |
272 | init_llist_head(&arb->requests); | 345 | init_llist_head(&arb->requests); |
@@ -324,6 +397,61 @@ init_fail: | |||
324 | return err; | 397 | return err; |
325 | } | 398 | } |
326 | 399 | ||
400 | void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm) | ||
401 | { | ||
402 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
403 | |||
404 | nvgpu_clk_arb_set_global_alarm(g, alarm); | ||
405 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
406 | } | ||
407 | |||
408 | static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm) | ||
409 | { | ||
410 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
411 | |||
412 | u64 current_mask; | ||
413 | u32 refcnt; | ||
414 | u32 alarm_mask; | ||
415 | u64 new_mask; | ||
416 | |||
417 | do { | ||
418 | current_mask = atomic64_read(&arb->alarm_mask); | ||
419 | /* atomic operations are strong so they do not need masks */ | ||
420 | |||
421 | refcnt = ((u32) (current_mask >> 32)) + 1; | ||
422 | alarm_mask = (u32) (current_mask & ~alarm); | ||
423 | new_mask = ((u64) refcnt << 32) | alarm_mask; | ||
424 | |||
425 | } while (unlikely(current_mask != | ||
426 | (u64)atomic64_cmpxchg(&arb->alarm_mask, | ||
427 | current_mask, new_mask))); | ||
428 | } | ||
429 | |||
430 | static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm) | ||
431 | { | ||
432 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
433 | |||
434 | u64 current_mask; | ||
435 | u32 refcnt; | ||
436 | u32 alarm_mask; | ||
437 | u64 new_mask; | ||
438 | |||
439 | do { | ||
440 | current_mask = atomic64_read(&arb->alarm_mask); | ||
441 | /* atomic operations are strong so they do not need masks */ | ||
442 | |||
443 | refcnt = ((u32) (current_mask >> 32)) + 1; | ||
444 | alarm_mask = (u32) (current_mask & ~0) | alarm; | ||
445 | new_mask = ((u64) refcnt << 32) | alarm_mask; | ||
446 | |||
447 | } while (unlikely(current_mask != | ||
448 | (u64)atomic64_cmpxchg(&arb->alarm_mask, | ||
449 | current_mask, new_mask))); | ||
450 | |||
451 | nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm); | ||
452 | |||
453 | } | ||
454 | |||
327 | void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g) | 455 | void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g) |
328 | { | 456 | { |
329 | kfree(g->clk_arb); | 457 | kfree(g->clk_arb); |
@@ -339,6 +467,7 @@ static int nvgpu_clk_arb_install_fd(struct gk20a *g, | |||
339 | int fd; | 467 | int fd; |
340 | int err; | 468 | int err; |
341 | struct nvgpu_clk_dev *dev; | 469 | struct nvgpu_clk_dev *dev; |
470 | int status; | ||
342 | 471 | ||
343 | gk20a_dbg_fn(""); | 472 | gk20a_dbg_fn(""); |
344 | 473 | ||
@@ -346,6 +475,12 @@ static int nvgpu_clk_arb_install_fd(struct gk20a *g, | |||
346 | if (!dev) | 475 | if (!dev) |
347 | return -ENOMEM; | 476 | return -ENOMEM; |
348 | 477 | ||
478 | status = nvgpu_clk_notification_queue_alloc(&dev->queue, | ||
479 | DEFAULT_EVENT_NUMBER); | ||
480 | if (status < 0) | ||
481 | return status; | ||
482 | |||
483 | |||
349 | fd = get_unused_fd_flags(O_RDWR); | 484 | fd = get_unused_fd_flags(O_RDWR); |
350 | if (fd < 0) { | 485 | if (fd < 0) { |
351 | err = fd; | 486 | err = fd; |
@@ -364,9 +499,7 @@ static int nvgpu_clk_arb_install_fd(struct gk20a *g, | |||
364 | 499 | ||
365 | init_waitqueue_head(&dev->readout_wq); | 500 | init_waitqueue_head(&dev->readout_wq); |
366 | 501 | ||
367 | spin_lock_init(&dev->event_lock); | 502 | atomic_set(&dev->poll_mask, 0); |
368 | dev->event_status = 0; | ||
369 | dev->event_mask = ~0; | ||
370 | 503 | ||
371 | dev->session = session; | 504 | dev->session = session; |
372 | kref_init(&dev->refcount); | 505 | kref_init(&dev->refcount); |
@@ -465,7 +598,7 @@ void nvgpu_clk_arb_release_session(struct gk20a *g, | |||
465 | } | 598 | } |
466 | 599 | ||
467 | int nvgpu_clk_arb_install_event_fd(struct gk20a *g, | 600 | int nvgpu_clk_arb_install_event_fd(struct gk20a *g, |
468 | struct nvgpu_clk_session *session, int *event_fd) | 601 | struct nvgpu_clk_session *session, int *event_fd, u32 alarm_mask) |
469 | { | 602 | { |
470 | struct nvgpu_clk_arb *arb = g->clk_arb; | 603 | struct nvgpu_clk_arb *arb = g->clk_arb; |
471 | struct nvgpu_clk_dev *dev; | 604 | struct nvgpu_clk_dev *dev; |
@@ -477,6 +610,17 @@ int nvgpu_clk_arb_install_event_fd(struct gk20a *g, | |||
477 | if (fd < 0) | 610 | if (fd < 0) |
478 | return fd; | 611 | return fd; |
479 | 612 | ||
613 | /* TODO: alarm mask needs to be set to default value to prevent | ||
614 | * failures of legacy tests. This will be removed when sanity is | ||
615 | * updated | ||
616 | */ | ||
617 | if (alarm_mask) | ||
618 | atomic_set(&dev->enabled_mask, alarm_mask); | ||
619 | else | ||
620 | atomic_set(&dev->enabled_mask, EVENT(VF_UPDATE)); | ||
621 | |||
622 | dev->arb_queue_head = atomic_read(&arb->notification_queue.head); | ||
623 | |||
480 | spin_lock(&arb->users_lock); | 624 | spin_lock(&arb->users_lock); |
481 | list_add_tail_rcu(&dev->link, &arb->users); | 625 | list_add_tail_rcu(&dev->link, &arb->users); |
482 | spin_unlock(&arb->users_lock); | 626 | spin_unlock(&arb->users_lock); |
@@ -813,9 +957,14 @@ static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb) | |||
813 | smp_wmb(); | 957 | smp_wmb(); |
814 | xchg(&arb->current_vf_table, table); | 958 | xchg(&arb->current_vf_table, table); |
815 | 959 | ||
816 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
817 | exit_vf_table: | 960 | exit_vf_table: |
818 | 961 | ||
962 | if (status < 0) | ||
963 | nvgpu_clk_arb_set_global_alarm(g, | ||
964 | EVENT(ALARM_VF_TABLE_UPDATE_FAILED)); | ||
965 | |||
966 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
967 | |||
819 | return status; | 968 | return status; |
820 | } | 969 | } |
821 | 970 | ||
@@ -838,6 +987,11 @@ static void nvgpu_clk_arb_run_vf_table_cb(struct work_struct *work) | |||
838 | if (err) { | 987 | if (err) { |
839 | gk20a_err(dev_from_gk20a(g), | 988 | gk20a_err(dev_from_gk20a(g), |
840 | "failed to cache VF table"); | 989 | "failed to cache VF table"); |
990 | nvgpu_clk_arb_set_global_alarm(g, | ||
991 | EVENT(ALARM_VF_TABLE_UPDATE_FAILED)); | ||
992 | |||
993 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
994 | |||
841 | return; | 995 | return; |
842 | } | 996 | } |
843 | nvgpu_clk_arb_update_vf_table(arb); | 997 | nvgpu_clk_arb_update_vf_table(arb); |
@@ -859,10 +1013,13 @@ static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | |||
859 | bool mclk_set, gpc2clk_set; | 1013 | bool mclk_set, gpc2clk_set; |
860 | u32 nuvmin, nuvmin_sram; | 1014 | u32 nuvmin, nuvmin_sram; |
861 | 1015 | ||
1016 | u32 alarms_notified = 0; | ||
1017 | u32 current_alarm; | ||
862 | int status = 0; | 1018 | int status = 0; |
863 | 1019 | ||
864 | /* Temporary variables for checking target frequency */ | 1020 | /* Temporary variables for checking target frequency */ |
865 | u16 gpc2clk_target, sys2clk_target, xbar2clk_target, mclk_target; | 1021 | u16 gpc2clk_target, sys2clk_target, xbar2clk_target, mclk_target; |
1022 | u16 gpc2clk_session_target, mclk_session_target; | ||
866 | 1023 | ||
867 | #ifdef CONFIG_DEBUG_FS | 1024 | #ifdef CONFIG_DEBUG_FS |
868 | u64 t0, t1; | 1025 | u64 t0, t1; |
@@ -872,6 +1029,10 @@ static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | |||
872 | 1029 | ||
873 | gk20a_dbg_fn(""); | 1030 | gk20a_dbg_fn(""); |
874 | 1031 | ||
1032 | /* bail out if gpu is down */ | ||
1033 | if (atomic_read(&arb->alarm_mask) & EVENT(ALARM_GPU_LOST)) | ||
1034 | goto exit_arb; | ||
1035 | |||
875 | #ifdef CONFIG_DEBUG_FS | 1036 | #ifdef CONFIG_DEBUG_FS |
876 | g->ops.read_ptimer(g, &t0); | 1037 | g->ops.read_ptimer(g, &t0); |
877 | #endif | 1038 | #endif |
@@ -937,6 +1098,10 @@ static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | |||
937 | 1098 | ||
938 | sys2clk_target = 0; | 1099 | sys2clk_target = 0; |
939 | xbar2clk_target = 0; | 1100 | xbar2clk_target = 0; |
1101 | |||
1102 | gpc2clk_session_target = gpc2clk_target; | ||
1103 | mclk_session_target = mclk_target; | ||
1104 | |||
940 | /* Query the table for the closest vf point to program */ | 1105 | /* Query the table for the closest vf point to program */ |
941 | pstate = nvgpu_clk_arb_find_vf_point(arb, &gpc2clk_target, | 1106 | pstate = nvgpu_clk_arb_find_vf_point(arb, &gpc2clk_target, |
942 | &sys2clk_target, &xbar2clk_target, &mclk_target, &voltuv, | 1107 | &sys2clk_target, &xbar2clk_target, &mclk_target, &voltuv, |
@@ -949,6 +1114,11 @@ static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | |||
949 | goto exit_arb; | 1114 | goto exit_arb; |
950 | } | 1115 | } |
951 | 1116 | ||
1117 | if ((gpc2clk_target < gpc2clk_session_target) || | ||
1118 | (mclk_target < mclk_session_target)) | ||
1119 | nvgpu_clk_arb_set_global_alarm(g, | ||
1120 | EVENT(ALARM_TARGET_VF_NOT_POSSIBLE)); | ||
1121 | |||
952 | if ((arb->actual->gpc2clk == gpc2clk_target) && | 1122 | if ((arb->actual->gpc2clk == gpc2clk_target) && |
953 | (arb->actual->mclk == mclk_target) && | 1123 | (arb->actual->mclk == mclk_target) && |
954 | (arb->voltuv_actual == voltuv)) { | 1124 | (arb->voltuv_actual == voltuv)) { |
@@ -1044,6 +1214,9 @@ static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | |||
1044 | /* Unlock pstate change for PG */ | 1214 | /* Unlock pstate change for PG */ |
1045 | mutex_unlock(&arb->pstate_lock); | 1215 | mutex_unlock(&arb->pstate_lock); |
1046 | 1216 | ||
1217 | /* VF Update complete */ | ||
1218 | nvgpu_clk_arb_set_global_alarm(g, EVENT(VF_UPDATE)); | ||
1219 | |||
1047 | wake_up_interruptible(&arb->request_wq); | 1220 | wake_up_interruptible(&arb->request_wq); |
1048 | 1221 | ||
1049 | #ifdef CONFIG_DEBUG_FS | 1222 | #ifdef CONFIG_DEBUG_FS |
@@ -1081,10 +1254,14 @@ static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | |||
1081 | #endif | 1254 | #endif |
1082 | 1255 | ||
1083 | exit_arb: | 1256 | exit_arb: |
1084 | if (status < 0) | 1257 | if (status < 0) { |
1085 | gk20a_err(dev_from_gk20a(g), | 1258 | gk20a_err(dev_from_gk20a(g), |
1086 | "Error in arbiter update"); | 1259 | "Error in arbiter update"); |
1260 | nvgpu_clk_arb_set_global_alarm(g, | ||
1261 | EVENT(ALARM_CLOCK_ARBITER_FAILED)); | ||
1262 | } | ||
1087 | 1263 | ||
1264 | current_alarm = (u32) atomic64_read(&arb->alarm_mask); | ||
1088 | /* notify completion for all requests */ | 1265 | /* notify completion for all requests */ |
1089 | head = llist_del_all(&arb->requests); | 1266 | head = llist_del_all(&arb->requests); |
1090 | llist_for_each_entry_safe(dev, tmp, head, node) { | 1267 | llist_for_each_entry_safe(dev, tmp, head, node) { |
@@ -1093,118 +1270,133 @@ exit_arb: | |||
1093 | kref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | 1270 | kref_put(&dev->refcount, nvgpu_clk_arb_free_fd); |
1094 | } | 1271 | } |
1095 | 1272 | ||
1273 | atomic_set(&arb->notification_queue.head, | ||
1274 | atomic_read(&arb->notification_queue.tail)); | ||
1096 | /* notify event for all users */ | 1275 | /* notify event for all users */ |
1097 | rcu_read_lock(); | 1276 | rcu_read_lock(); |
1098 | list_for_each_entry_rcu(dev, &arb->users, link) { | 1277 | list_for_each_entry_rcu(dev, &arb->users, link) { |
1099 | spin_lock(&dev->event_lock); | 1278 | alarms_notified |= |
1100 | dev->event_status |= (1UL << NVGPU_GPU_EVENT_VF_UPDATE); | 1279 | nvgpu_clk_arb_notify(dev, arb->actual, current_alarm); |
1101 | spin_unlock(&dev->event_lock); | ||
1102 | wake_up_interruptible(&dev->readout_wq); | ||
1103 | } | 1280 | } |
1104 | rcu_read_unlock(); | 1281 | rcu_read_unlock(); |
1105 | } | ||
1106 | |||
1107 | int nvgpu_clk_arb_commit_request_fd(struct gk20a *g, | ||
1108 | struct nvgpu_clk_session *session, int request_fd) | ||
1109 | { | ||
1110 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1111 | struct nvgpu_clk_dev *dev; | ||
1112 | struct fd fd; | ||
1113 | int err = 0; | ||
1114 | 1282 | ||
1115 | gk20a_dbg_fn(""); | 1283 | /* clear alarms */ |
1284 | nvgpu_clk_arb_clear_global_alarm(g, alarms_notified & | ||
1285 | ~EVENT(ALARM_GPU_LOST)); | ||
1286 | } | ||
1116 | 1287 | ||
1117 | fd = fdget(request_fd); | 1288 | static void nvgpu_clk_arb_queue_notification(struct gk20a *g, |
1289 | struct nvgpu_clk_notification_queue *queue, | ||
1290 | u32 alarm_mask) { | ||
1118 | 1291 | ||
1119 | if (!fd.file) | 1292 | u32 queue_index; |
1120 | return -EINVAL; | 1293 | u64 timestamp; |
1121 | |||
1122 | dev = (struct nvgpu_clk_dev *) fd.file->private_data; | ||
1123 | 1294 | ||
1124 | if (!dev || dev->session != session) { | 1295 | queue_index = (atomic_inc_return(&queue->tail)) % queue->size; |
1125 | err = -EINVAL; | 1296 | /* get current timestamp */ |
1126 | goto fdput_fd; | 1297 | timestamp = (u64) get_cycles(); |
1127 | } | ||
1128 | kref_get(&dev->refcount); | ||
1129 | llist_add(&dev->node, &session->targets); | ||
1130 | 1298 | ||
1131 | queue_work(arb->update_work_queue, &arb->update_fn_work); | 1299 | queue->notifications[queue_index].timestamp = timestamp; |
1300 | queue->notifications[queue_index].notification = alarm_mask; | ||
1132 | 1301 | ||
1133 | fdput_fd: | ||
1134 | fdput(fd); | ||
1135 | return err; | ||
1136 | } | 1302 | } |
1137 | 1303 | ||
1138 | static inline int __pending_event(struct nvgpu_clk_dev *dev, | 1304 | static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, |
1139 | struct nvgpu_gpu_event_info *info) | 1305 | struct nvgpu_clk_arb_target *target, |
1140 | { | 1306 | u32 alarm) { |
1141 | struct gk20a *g = dev->session->g; | ||
1142 | u32 status; | ||
1143 | |||
1144 | spin_lock(&dev->event_lock); | ||
1145 | status = dev->event_status & dev->event_mask; | ||
1146 | if (status && info) | ||
1147 | { | ||
1148 | /* TODO: retrieve oldest event_id based on timestamp */ | ||
1149 | info->event_id = ffs(status) - 1; | ||
1150 | g->ops.read_ptimer(g, &info->timestamp); | ||
1151 | 1307 | ||
1152 | dev->event_status &= ~(1UL << info->event_id); | 1308 | struct nvgpu_clk_session *session = dev->session; |
1153 | } | 1309 | struct nvgpu_clk_arb *arb = session->g->clk_arb; |
1154 | spin_unlock(&dev->event_lock); | 1310 | struct nvgpu_clk_notification *notification; |
1155 | return status; | 1311 | |
1156 | } | 1312 | u32 queue_alarm_mask = 0; |
1313 | u32 enabled_mask = 0; | ||
1314 | u32 new_alarms_reported = 0; | ||
1315 | u32 poll_mask = 0; | ||
1316 | u32 tail, head; | ||
1317 | u32 queue_index; | ||
1318 | size_t size; | ||
1319 | int index; | ||
1157 | 1320 | ||
1158 | static unsigned int nvgpu_clk_arb_poll_completion_dev(struct file *filp, | 1321 | enabled_mask = atomic_read(&dev->enabled_mask); |
1159 | poll_table *wait) | 1322 | size = arb->notification_queue.size; |
1160 | { | ||
1161 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1162 | 1323 | ||
1163 | gk20a_dbg_fn(""); | 1324 | /* queue global arbiter notifications in buffer */ |
1325 | do { | ||
1326 | tail = atomic_read(&arb->notification_queue.tail); | ||
1327 | /* copy items to the queue */ | ||
1328 | queue_index = atomic_read(&dev->queue.tail); | ||
1329 | head = dev->arb_queue_head; | ||
1330 | head = (tail - head) < arb->notification_queue.size ? | ||
1331 | head : tail - arb->notification_queue.size; | ||
1332 | |||
1333 | for (index = head; _WRAPGTEQ(tail, index); index++) { | ||
1334 | u32 alarm_detected; | ||
1335 | |||
1336 | notification = &arb->notification_queue. | ||
1337 | notifications[(index+1) % size]; | ||
1338 | alarm_detected = | ||
1339 | ACCESS_ONCE(notification->notification); | ||
1340 | |||
1341 | if (!(enabled_mask & alarm_detected)) | ||
1342 | continue; | ||
1343 | |||
1344 | queue_index++; | ||
1345 | dev->queue.notifications[ | ||
1346 | queue_index % dev->queue.size].timestamp = | ||
1347 | ACCESS_ONCE(notification->timestamp); | ||
1348 | |||
1349 | dev->queue.notifications[ | ||
1350 | queue_index % dev->queue.size].notification = | ||
1351 | alarm_detected; | ||
1352 | |||
1353 | queue_alarm_mask |= alarm_detected; | ||
1354 | } | ||
1355 | } while (unlikely(atomic_read(&arb->notification_queue.tail) != | ||
1356 | (int)tail)); | ||
1164 | 1357 | ||
1165 | poll_wait(filp, &dev->readout_wq, wait); | 1358 | atomic_set(&dev->queue.tail, queue_index); |
1166 | return atomic_xchg(&dev->poll_mask, 0); | 1359 | /* update the last notification we processed from global queue */ |
1167 | } | ||
1168 | 1360 | ||
1169 | static unsigned int nvgpu_clk_arb_poll_event_dev(struct file *filp, | 1361 | dev->arb_queue_head = tail; |
1170 | poll_table *wait) | ||
1171 | { | ||
1172 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1173 | 1362 | ||
1174 | gk20a_dbg_fn(""); | 1363 | /* Check if current session targets are met */ |
1364 | if (enabled_mask & EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)) { | ||
1365 | if ((target->gpc2clk < session->target->gpc2clk) | ||
1366 | || (target->mclk < session->target->mclk)) { | ||
1175 | 1367 | ||
1176 | poll_wait(filp, &dev->readout_wq, wait); | 1368 | poll_mask |= (POLLIN | POLLPRI); |
1177 | return __pending_event(dev, NULL); | 1369 | nvgpu_clk_arb_queue_notification(arb->g, &dev->queue, |
1178 | } | 1370 | EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)); |
1371 | } | ||
1372 | } | ||
1179 | 1373 | ||
1180 | static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, | 1374 | /* Check if there is a new VF update */ |
1181 | char __user *buf, size_t size, loff_t *off) | 1375 | if (queue_alarm_mask & EVENT(VF_UPDATE)) |
1182 | { | 1376 | poll_mask |= (POLLIN | POLLRDNORM); |
1183 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1184 | struct nvgpu_gpu_event_info info; | ||
1185 | int err; | ||
1186 | 1377 | ||
1187 | gk20a_dbg(gpu_dbg_fn, "filp=%p buf=%p size=%zu", filp, buf, size); | 1378 | /* Notify sticky alarms that were not reported on previous run*/ |
1379 | new_alarms_reported = (queue_alarm_mask | | ||
1380 | (alarm & ~dev->alarms_reported & queue_alarm_mask)); | ||
1188 | 1381 | ||
1189 | if (size < sizeof(info)) | 1382 | if (new_alarms_reported & ~LOCAL_ALARM_MASK) { |
1190 | return 0; | 1383 | /* check that we are not re-reporting */ |
1384 | if (new_alarms_reported & EVENT(ALARM_GPU_LOST)) | ||
1385 | poll_mask |= POLLHUP; | ||
1191 | 1386 | ||
1192 | memset(&info, 0, sizeof(info)); | 1387 | poll_mask |= (POLLIN | POLLPRI); |
1193 | while (!__pending_event(dev, &info)) { | 1388 | /* On next run do not report global alarms that were already |
1194 | if (filp->f_flags & O_NONBLOCK) | 1389 | * reported, but report SHUTDOWN always */ |
1195 | return -EAGAIN; | 1390 | dev->alarms_reported = new_alarms_reported & ~LOCAL_ALARM_MASK & |
1196 | err = wait_event_interruptible(dev->readout_wq, | 1391 | ~EVENT(ALARM_GPU_LOST); |
1197 | __pending_event(dev, &info)); | ||
1198 | if (err) | ||
1199 | return err; | ||
1200 | } | 1392 | } |
1201 | 1393 | ||
1202 | if (copy_to_user(buf, &info, sizeof(info))) | 1394 | if (poll_mask) { |
1203 | return -EFAULT; | 1395 | atomic_set(&dev->poll_mask, poll_mask); |
1204 | 1396 | wake_up_interruptible_all(&dev->readout_wq); | |
1205 | *off += sizeof(info); | 1397 | } |
1206 | 1398 | ||
1207 | return sizeof(info); | 1399 | return new_alarms_reported; |
1208 | } | 1400 | } |
1209 | 1401 | ||
1210 | static int nvgpu_clk_arb_set_event_filter(struct nvgpu_clk_dev *dev, | 1402 | static int nvgpu_clk_arb_set_event_filter(struct nvgpu_clk_dev *dev, |
@@ -1224,10 +1416,8 @@ static int nvgpu_clk_arb_set_event_filter(struct nvgpu_clk_dev *dev, | |||
1224 | args->size * sizeof(u32))) | 1416 | args->size * sizeof(u32))) |
1225 | return -EFAULT; | 1417 | return -EFAULT; |
1226 | 1418 | ||
1227 | spin_lock(&dev->event_lock); | 1419 | /* update alarm mask */ |
1228 | /* update event mask */ | 1420 | atomic_set(&dev->enabled_mask, mask); |
1229 | dev->event_mask = mask; | ||
1230 | spin_unlock(&dev->event_lock); | ||
1231 | 1421 | ||
1232 | return 0; | 1422 | return 0; |
1233 | } | 1423 | } |
@@ -1271,6 +1461,106 @@ long nvgpu_clk_arb_ioctl_event_dev(struct file *filp, unsigned int cmd, | |||
1271 | return err; | 1461 | return err; |
1272 | } | 1462 | } |
1273 | 1463 | ||
1464 | int nvgpu_clk_arb_commit_request_fd(struct gk20a *g, | ||
1465 | struct nvgpu_clk_session *session, int request_fd) | ||
1466 | { | ||
1467 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1468 | struct nvgpu_clk_dev *dev; | ||
1469 | struct fd fd; | ||
1470 | int err = 0; | ||
1471 | |||
1472 | gk20a_dbg_fn(""); | ||
1473 | |||
1474 | fd = fdget(request_fd); | ||
1475 | if (!fd.file) | ||
1476 | return -EINVAL; | ||
1477 | |||
1478 | if (fd.file->f_op != &completion_dev_ops) { | ||
1479 | err = -EINVAL; | ||
1480 | goto fdput_fd; | ||
1481 | } | ||
1482 | |||
1483 | dev = (struct nvgpu_clk_dev *) fd.file->private_data; | ||
1484 | |||
1485 | if (!dev || dev->session != session) { | ||
1486 | err = -EINVAL; | ||
1487 | goto fdput_fd; | ||
1488 | } | ||
1489 | kref_get(&dev->refcount); | ||
1490 | llist_add(&dev->node, &session->targets); | ||
1491 | |||
1492 | queue_work(arb->update_work_queue, &arb->update_fn_work); | ||
1493 | |||
1494 | fdput_fd: | ||
1495 | fdput(fd); | ||
1496 | return err; | ||
1497 | } | ||
1498 | |||
1499 | static inline u32 __pending_event(struct nvgpu_clk_dev *dev, | ||
1500 | struct nvgpu_gpu_event_info *info) { | ||
1501 | |||
1502 | u32 tail, head; | ||
1503 | u32 events = 0; | ||
1504 | struct nvgpu_clk_notification *p_notif; | ||
1505 | |||
1506 | tail = atomic_read(&dev->queue.tail); | ||
1507 | head = atomic_read(&dev->queue.head); | ||
1508 | |||
1509 | head = (tail - head) < dev->queue.size ? head : tail - dev->queue.size; | ||
1510 | |||
1511 | if (_WRAPGTEQ(tail, head) && info) { | ||
1512 | head++; | ||
1513 | p_notif = &dev->queue.notifications[head % dev->queue.size]; | ||
1514 | events |= p_notif->notification; | ||
1515 | info->event_id = ffs(events) - 1; | ||
1516 | info->timestamp = p_notif->timestamp; | ||
1517 | atomic_set(&dev->queue.head, head); | ||
1518 | } | ||
1519 | |||
1520 | return events; | ||
1521 | } | ||
1522 | |||
1523 | static ssize_t nvgpu_clk_arb_read_event_dev(struct file *filp, char __user *buf, | ||
1524 | size_t size, loff_t *off) | ||
1525 | { | ||
1526 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1527 | struct nvgpu_gpu_event_info info; | ||
1528 | ssize_t err; | ||
1529 | |||
1530 | gk20a_dbg_fn("filp=%p, buf=%p, size=%zu", filp, buf, size); | ||
1531 | |||
1532 | if ((size - *off) < sizeof(info)) | ||
1533 | return 0; | ||
1534 | |||
1535 | memset(&info, 0, sizeof(info)); | ||
1536 | /* Get the oldest event from the queue */ | ||
1537 | while (!__pending_event(dev, &info)) { | ||
1538 | if (filp->f_flags & O_NONBLOCK) | ||
1539 | return -EAGAIN; | ||
1540 | err = wait_event_interruptible(dev->readout_wq, | ||
1541 | __pending_event(dev, &info)); | ||
1542 | if (err) | ||
1543 | return err; | ||
1544 | if (info.timestamp) | ||
1545 | break; | ||
1546 | } | ||
1547 | |||
1548 | if (copy_to_user(buf + *off, &info, sizeof(info))) | ||
1549 | return -EFAULT; | ||
1550 | |||
1551 | return sizeof(info); | ||
1552 | } | ||
1553 | |||
1554 | static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait) | ||
1555 | { | ||
1556 | struct nvgpu_clk_dev *dev = filp->private_data; | ||
1557 | |||
1558 | gk20a_dbg_fn(""); | ||
1559 | |||
1560 | poll_wait(filp, &dev->readout_wq, wait); | ||
1561 | return atomic_xchg(&dev->poll_mask, 0); | ||
1562 | } | ||
1563 | |||
1274 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, | 1564 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, |
1275 | struct file *filp) | 1565 | struct file *filp) |
1276 | { | 1566 | { |
@@ -1305,6 +1595,8 @@ static int nvgpu_clk_arb_release_event_dev(struct inode *inode, | |||
1305 | 1595 | ||
1306 | synchronize_rcu(); | 1596 | synchronize_rcu(); |
1307 | kref_put(&session->refcount, nvgpu_clk_arb_free_session); | 1597 | kref_put(&session->refcount, nvgpu_clk_arb_free_session); |
1598 | |||
1599 | nvgpu_clk_notification_queue_free(&dev->queue); | ||
1308 | kfree(dev); | 1600 | kfree(dev); |
1309 | 1601 | ||
1310 | return 0; | 1602 | return 0; |
@@ -1320,10 +1612,14 @@ int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session, | |||
1320 | gk20a_dbg_fn("domain=0x%08x target_mhz=%u", api_domain, target_mhz); | 1612 | gk20a_dbg_fn("domain=0x%08x target_mhz=%u", api_domain, target_mhz); |
1321 | 1613 | ||
1322 | fd = fdget(request_fd); | 1614 | fd = fdget(request_fd); |
1323 | |||
1324 | if (!fd.file) | 1615 | if (!fd.file) |
1325 | return -EINVAL; | 1616 | return -EINVAL; |
1326 | 1617 | ||
1618 | if (fd.file->f_op != &completion_dev_ops) { | ||
1619 | err = -EINVAL; | ||
1620 | goto fdput_fd; | ||
1621 | } | ||
1622 | |||
1327 | dev = fd.file->private_data; | 1623 | dev = fd.file->private_data; |
1328 | if (!dev || dev->session != session) { | 1624 | if (!dev || dev->session != session) { |
1329 | err = -EINVAL; | 1625 | err = -EINVAL; |
@@ -1599,7 +1895,7 @@ find_exit: | |||
1599 | /* noise unaware vmin */ | 1895 | /* noise unaware vmin */ |
1600 | *nuvmin = mclk_voltuv; | 1896 | *nuvmin = mclk_voltuv; |
1601 | *nuvmin_sram = mclk_voltuv_sram; | 1897 | *nuvmin_sram = mclk_voltuv_sram; |
1602 | *gpc2clk = gpc2clk_target; | 1898 | *gpc2clk = gpc2clk_target < *gpc2clk ? gpc2clk_target : *gpc2clk; |
1603 | *mclk = mclk_target; | 1899 | *mclk = mclk_target; |
1604 | return pstate; | 1900 | return pstate; |
1605 | } | 1901 | } |
diff --git a/drivers/gpu/nvgpu/clk/clk_arb.h b/drivers/gpu/nvgpu/clk/clk_arb.h index 700804b3..c7dc8d19 100644 --- a/drivers/gpu/nvgpu/clk/clk_arb.h +++ b/drivers/gpu/nvgpu/clk/clk_arb.h | |||
@@ -56,7 +56,7 @@ int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session, | |||
56 | u32 api_domain, u16 *target_mhz); | 56 | u32 api_domain, u16 *target_mhz); |
57 | 57 | ||
58 | int nvgpu_clk_arb_install_event_fd(struct gk20a *g, | 58 | int nvgpu_clk_arb_install_event_fd(struct gk20a *g, |
59 | struct nvgpu_clk_session *session, int *event_fd); | 59 | struct nvgpu_clk_session *session, int *event_fd, u32 alarm_mask); |
60 | 60 | ||
61 | int nvgpu_clk_arb_install_request_fd(struct gk20a *g, | 61 | int nvgpu_clk_arb_install_request_fd(struct gk20a *g, |
62 | struct nvgpu_clk_session *session, int *event_fd); | 62 | struct nvgpu_clk_session *session, int *event_fd); |
@@ -67,5 +67,6 @@ int nvgpu_clk_arb_get_current_pstate(struct gk20a *g); | |||
67 | 67 | ||
68 | void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock); | 68 | void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock); |
69 | 69 | ||
70 | void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm); | ||
70 | #endif /* _CLK_ARB_H_ */ | 71 | #endif /* _CLK_ARB_H_ */ |
71 | 72 | ||