diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common')
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/clk_arb.c | 1522 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/clk_arb_linux.h | 163 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/ioctl_clk_arb.c | 1 |
3 files changed, 0 insertions, 1686 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/clk_arb.c b/drivers/gpu/nvgpu/common/linux/clk_arb.c deleted file mode 100644 index 2a6278e8..00000000 --- a/drivers/gpu/nvgpu/common/linux/clk_arb.c +++ /dev/null | |||
@@ -1,1522 +0,0 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This software is licensed under the terms of the GNU General Public | ||
5 | * License version 2, as published by the Free Software Foundation, and | ||
6 | * may be copied, distributed, and modified under those terms. | ||
7 | * | ||
8 | * This program is distributed in the hope that it will be useful, | ||
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
11 | * GNU General Public License for more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #include <nvgpu/bitops.h> | ||
18 | #include <nvgpu/lock.h> | ||
19 | #include <nvgpu/kmem.h> | ||
20 | #include <nvgpu/atomic.h> | ||
21 | #include <nvgpu/bug.h> | ||
22 | #include <nvgpu/kref.h> | ||
23 | #include <nvgpu/log.h> | ||
24 | #include <nvgpu/barrier.h> | ||
25 | #include <nvgpu/cond.h> | ||
26 | #include <nvgpu/list.h> | ||
27 | #include <nvgpu/clk_arb.h> | ||
28 | |||
29 | #include "gk20a/gk20a.h" | ||
30 | #include "clk/clk.h" | ||
31 | #include "clk_arb_linux.h" | ||
32 | #include "pstate/pstate.h" | ||
33 | #include "lpwr/lpwr.h" | ||
34 | #include "volt/volt.h" | ||
35 | |||
36 | int nvgpu_clk_notification_queue_alloc(struct gk20a *g, | ||
37 | struct nvgpu_clk_notification_queue *queue, | ||
38 | size_t events_number) { | ||
39 | queue->notifications = nvgpu_kcalloc(g, events_number, | ||
40 | sizeof(struct nvgpu_clk_notification)); | ||
41 | if (!queue->notifications) | ||
42 | return -ENOMEM; | ||
43 | queue->size = events_number; | ||
44 | |||
45 | nvgpu_atomic_set(&queue->head, 0); | ||
46 | nvgpu_atomic_set(&queue->tail, 0); | ||
47 | |||
48 | return 0; | ||
49 | } | ||
50 | |||
51 | void nvgpu_clk_notification_queue_free(struct gk20a *g, | ||
52 | struct nvgpu_clk_notification_queue *queue) { | ||
53 | nvgpu_kfree(g, queue->notifications); | ||
54 | queue->size = 0; | ||
55 | nvgpu_atomic_set(&queue->head, 0); | ||
56 | nvgpu_atomic_set(&queue->tail, 0); | ||
57 | } | ||
58 | |||
59 | static void nvgpu_clk_arb_queue_notification(struct gk20a *g, | ||
60 | struct nvgpu_clk_notification_queue *queue, | ||
61 | u32 alarm_mask) { | ||
62 | |||
63 | u32 queue_index; | ||
64 | u64 timestamp; | ||
65 | |||
66 | queue_index = (nvgpu_atomic_inc_return(&queue->tail)) % queue->size; | ||
67 | /* get current timestamp */ | ||
68 | timestamp = (u64) sched_clock(); | ||
69 | |||
70 | queue->notifications[queue_index].timestamp = timestamp; | ||
71 | queue->notifications[queue_index].notification = alarm_mask; | ||
72 | |||
73 | } | ||
74 | |||
75 | static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm) | ||
76 | { | ||
77 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
78 | |||
79 | u64 current_mask; | ||
80 | u32 refcnt; | ||
81 | u32 alarm_mask; | ||
82 | u64 new_mask; | ||
83 | |||
84 | do { | ||
85 | current_mask = nvgpu_atomic64_read(&arb->alarm_mask); | ||
86 | /* atomic operations are strong so they do not need masks */ | ||
87 | |||
88 | refcnt = ((u32) (current_mask >> 32)) + 1; | ||
89 | alarm_mask = (u32) (current_mask & ~0) | alarm; | ||
90 | new_mask = ((u64) refcnt << 32) | alarm_mask; | ||
91 | |||
92 | } while (unlikely(current_mask != | ||
93 | (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask, | ||
94 | current_mask, new_mask))); | ||
95 | |||
96 | nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm); | ||
97 | } | ||
98 | |||
99 | |||
100 | static int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb) | ||
101 | { | ||
102 | struct gk20a *g = arb->g; | ||
103 | struct nvgpu_clk_vf_table *table; | ||
104 | |||
105 | u32 i, j; | ||
106 | int status = -EINVAL; | ||
107 | u32 gpc2clk_voltuv = 0, mclk_voltuv = 0; | ||
108 | u32 gpc2clk_voltuv_sram = 0, mclk_voltuv_sram = 0; | ||
109 | u16 clk_cur; | ||
110 | u32 num_points; | ||
111 | |||
112 | struct clk_set_info *p5_info, *p0_info; | ||
113 | |||
114 | |||
115 | table = NV_ACCESS_ONCE(arb->current_vf_table); | ||
116 | /* make flag visible when all data has resolved in the tables */ | ||
117 | nvgpu_smp_rmb(); | ||
118 | |||
119 | table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] : | ||
120 | &arb->vf_table_pool[0]; | ||
121 | |||
122 | /* Get allowed memory ranges */ | ||
123 | if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
124 | &arb->gpc2clk_min, | ||
125 | &arb->gpc2clk_max) < 0) { | ||
126 | nvgpu_err(g, "failed to fetch GPC2CLK range"); | ||
127 | goto exit_vf_table; | ||
128 | } | ||
129 | if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_MCLK, | ||
130 | &arb->mclk_min, | ||
131 | &arb->mclk_max) < 0) { | ||
132 | nvgpu_err(g, "failed to fetch MCLK range"); | ||
133 | goto exit_vf_table; | ||
134 | } | ||
135 | |||
136 | table->gpc2clk_num_points = MAX_F_POINTS; | ||
137 | table->mclk_num_points = MAX_F_POINTS; | ||
138 | |||
139 | if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
140 | &table->gpc2clk_num_points, arb->gpc2clk_f_points)) { | ||
141 | nvgpu_err(g, "failed to fetch GPC2CLK frequency points"); | ||
142 | goto exit_vf_table; | ||
143 | } | ||
144 | |||
145 | if (clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_MCLK, | ||
146 | &table->mclk_num_points, arb->mclk_f_points)) { | ||
147 | nvgpu_err(g, "failed to fetch MCLK frequency points"); | ||
148 | goto exit_vf_table; | ||
149 | } | ||
150 | if (!table->mclk_num_points || !table->gpc2clk_num_points) { | ||
151 | nvgpu_err(g, "empty queries to f points mclk %d gpc2clk %d", | ||
152 | table->mclk_num_points, table->gpc2clk_num_points); | ||
153 | status = -EINVAL; | ||
154 | goto exit_vf_table; | ||
155 | } | ||
156 | |||
157 | memset(table->mclk_points, 0, | ||
158 | table->mclk_num_points*sizeof(struct nvgpu_clk_vf_point)); | ||
159 | memset(table->gpc2clk_points, 0, | ||
160 | table->gpc2clk_num_points*sizeof(struct nvgpu_clk_vf_point)); | ||
161 | |||
162 | p5_info = pstate_get_clk_set_info(g, | ||
163 | CTRL_PERF_PSTATE_P5, clkwhich_mclk); | ||
164 | if (!p5_info) { | ||
165 | nvgpu_err(g, "failed to get MCLK P5 info"); | ||
166 | goto exit_vf_table; | ||
167 | } | ||
168 | p0_info = pstate_get_clk_set_info(g, | ||
169 | CTRL_PERF_PSTATE_P0, clkwhich_mclk); | ||
170 | if (!p0_info) { | ||
171 | nvgpu_err(g, "failed to get MCLK P0 info"); | ||
172 | goto exit_vf_table; | ||
173 | } | ||
174 | |||
175 | for (i = 0, j = 0, num_points = 0, clk_cur = 0; | ||
176 | i < table->mclk_num_points; i++) { | ||
177 | |||
178 | if ((arb->mclk_f_points[i] >= arb->mclk_min) && | ||
179 | (arb->mclk_f_points[i] <= arb->mclk_max) && | ||
180 | (arb->mclk_f_points[i] != clk_cur)) { | ||
181 | |||
182 | table->mclk_points[j].mem_mhz = arb->mclk_f_points[i]; | ||
183 | mclk_voltuv = mclk_voltuv_sram = 0; | ||
184 | |||
185 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK, | ||
186 | &table->mclk_points[j].mem_mhz, &mclk_voltuv, | ||
187 | CTRL_VOLT_DOMAIN_LOGIC); | ||
188 | if (status < 0) { | ||
189 | nvgpu_err(g, | ||
190 | "failed to get MCLK LOGIC voltage"); | ||
191 | goto exit_vf_table; | ||
192 | } | ||
193 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK, | ||
194 | &table->mclk_points[j].mem_mhz, | ||
195 | &mclk_voltuv_sram, | ||
196 | CTRL_VOLT_DOMAIN_SRAM); | ||
197 | if (status < 0) { | ||
198 | nvgpu_err(g, "failed to get MCLK SRAM voltage"); | ||
199 | goto exit_vf_table; | ||
200 | } | ||
201 | |||
202 | table->mclk_points[j].uvolt = mclk_voltuv; | ||
203 | table->mclk_points[j].uvolt_sram = mclk_voltuv_sram; | ||
204 | clk_cur = table->mclk_points[j].mem_mhz; | ||
205 | |||
206 | if ((clk_cur >= p5_info->min_mhz) && | ||
207 | (clk_cur <= p5_info->max_mhz)) | ||
208 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
209 | &table->mclk_points[j], | ||
210 | CTRL_PERF_PSTATE_P5); | ||
211 | if ((clk_cur >= p0_info->min_mhz) && | ||
212 | (clk_cur <= p0_info->max_mhz)) | ||
213 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
214 | &table->mclk_points[j], | ||
215 | CTRL_PERF_PSTATE_P0); | ||
216 | |||
217 | j++; | ||
218 | num_points++; | ||
219 | |||
220 | } | ||
221 | } | ||
222 | table->mclk_num_points = num_points; | ||
223 | |||
224 | p5_info = pstate_get_clk_set_info(g, | ||
225 | CTRL_PERF_PSTATE_P5, clkwhich_gpc2clk); | ||
226 | if (!p5_info) { | ||
227 | status = -EINVAL; | ||
228 | nvgpu_err(g, "failed to get GPC2CLK P5 info"); | ||
229 | goto exit_vf_table; | ||
230 | } | ||
231 | |||
232 | p0_info = pstate_get_clk_set_info(g, | ||
233 | CTRL_PERF_PSTATE_P0, clkwhich_gpc2clk); | ||
234 | if (!p0_info) { | ||
235 | status = -EINVAL; | ||
236 | nvgpu_err(g, "failed to get GPC2CLK P0 info"); | ||
237 | goto exit_vf_table; | ||
238 | } | ||
239 | |||
240 | /* GPC2CLK needs to be checked in two passes. The first determines the | ||
241 | * relationships between GPC2CLK, SYS2CLK and XBAR2CLK, while the | ||
242 | * second verifies that the clocks minimum is satisfied and sets | ||
243 | * the voltages | ||
244 | */ | ||
245 | for (i = 0, j = 0, num_points = 0, clk_cur = 0; | ||
246 | i < table->gpc2clk_num_points; i++) { | ||
247 | struct set_fll_clk setfllclk; | ||
248 | |||
249 | if ((arb->gpc2clk_f_points[i] >= arb->gpc2clk_min) && | ||
250 | (arb->gpc2clk_f_points[i] <= arb->gpc2clk_max) && | ||
251 | (arb->gpc2clk_f_points[i] != clk_cur)) { | ||
252 | |||
253 | table->gpc2clk_points[j].gpc_mhz = | ||
254 | arb->gpc2clk_f_points[i]; | ||
255 | setfllclk.gpc2clkmhz = arb->gpc2clk_f_points[i]; | ||
256 | status = clk_get_fll_clks(g, &setfllclk); | ||
257 | if (status < 0) { | ||
258 | nvgpu_err(g, | ||
259 | "failed to get GPC2CLK slave clocks"); | ||
260 | goto exit_vf_table; | ||
261 | } | ||
262 | |||
263 | table->gpc2clk_points[j].sys_mhz = | ||
264 | setfllclk.sys2clkmhz; | ||
265 | table->gpc2clk_points[j].xbar_mhz = | ||
266 | setfllclk.xbar2clkmhz; | ||
267 | |||
268 | clk_cur = table->gpc2clk_points[j].gpc_mhz; | ||
269 | |||
270 | if ((clk_cur >= p5_info->min_mhz) && | ||
271 | (clk_cur <= p5_info->max_mhz)) | ||
272 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
273 | &table->gpc2clk_points[j], | ||
274 | CTRL_PERF_PSTATE_P5); | ||
275 | if ((clk_cur >= p0_info->min_mhz) && | ||
276 | (clk_cur <= p0_info->max_mhz)) | ||
277 | VF_POINT_SET_PSTATE_SUPPORTED( | ||
278 | &table->gpc2clk_points[j], | ||
279 | CTRL_PERF_PSTATE_P0); | ||
280 | |||
281 | j++; | ||
282 | num_points++; | ||
283 | } | ||
284 | } | ||
285 | table->gpc2clk_num_points = num_points; | ||
286 | |||
287 | /* Second pass */ | ||
288 | for (i = 0, j = 0; i < table->gpc2clk_num_points; i++) { | ||
289 | |||
290 | u16 alt_gpc2clk = table->gpc2clk_points[i].gpc_mhz; | ||
291 | |||
292 | gpc2clk_voltuv = gpc2clk_voltuv_sram = 0; | ||
293 | |||
294 | /* Check sysclk */ | ||
295 | p5_info = pstate_get_clk_set_info(g, | ||
296 | VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]), | ||
297 | clkwhich_sys2clk); | ||
298 | if (!p5_info) { | ||
299 | status = -EINVAL; | ||
300 | nvgpu_err(g, "failed to get SYS2CLK P5 info"); | ||
301 | goto exit_vf_table; | ||
302 | } | ||
303 | |||
304 | /* sys2clk below clk min, need to find correct clock */ | ||
305 | if (table->gpc2clk_points[i].sys_mhz < p5_info->min_mhz) { | ||
306 | for (j = i + 1; j < table->gpc2clk_num_points; j++) { | ||
307 | |||
308 | if (table->gpc2clk_points[j].sys_mhz >= | ||
309 | p5_info->min_mhz) { | ||
310 | |||
311 | |||
312 | table->gpc2clk_points[i].sys_mhz = | ||
313 | p5_info->min_mhz; | ||
314 | |||
315 | alt_gpc2clk = alt_gpc2clk < | ||
316 | table->gpc2clk_points[j]. | ||
317 | gpc_mhz ? | ||
318 | table->gpc2clk_points[j]. | ||
319 | gpc_mhz : | ||
320 | alt_gpc2clk; | ||
321 | break; | ||
322 | } | ||
323 | } | ||
324 | /* no VF exists that satisfies condition */ | ||
325 | if (j == table->gpc2clk_num_points) { | ||
326 | nvgpu_err(g, "NO SYS2CLK VF point possible"); | ||
327 | status = -EINVAL; | ||
328 | goto exit_vf_table; | ||
329 | } | ||
330 | } | ||
331 | |||
332 | /* Check xbarclk */ | ||
333 | p5_info = pstate_get_clk_set_info(g, | ||
334 | VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]), | ||
335 | clkwhich_xbar2clk); | ||
336 | if (!p5_info) { | ||
337 | status = -EINVAL; | ||
338 | nvgpu_err(g, "failed to get SYS2CLK P5 info"); | ||
339 | goto exit_vf_table; | ||
340 | } | ||
341 | |||
342 | /* xbar2clk below clk min, need to find correct clock */ | ||
343 | if (table->gpc2clk_points[i].xbar_mhz < p5_info->min_mhz) { | ||
344 | for (j = i; j < table->gpc2clk_num_points; j++) { | ||
345 | if (table->gpc2clk_points[j].xbar_mhz >= | ||
346 | p5_info->min_mhz) { | ||
347 | |||
348 | table->gpc2clk_points[i].xbar_mhz = | ||
349 | p5_info->min_mhz; | ||
350 | |||
351 | alt_gpc2clk = alt_gpc2clk < | ||
352 | table->gpc2clk_points[j]. | ||
353 | gpc_mhz ? | ||
354 | table->gpc2clk_points[j]. | ||
355 | gpc_mhz : | ||
356 | alt_gpc2clk; | ||
357 | break; | ||
358 | } | ||
359 | } | ||
360 | /* no VF exists that satisfies condition */ | ||
361 | if (j == table->gpc2clk_num_points) { | ||
362 | status = -EINVAL; | ||
363 | nvgpu_err(g, "NO XBAR2CLK VF point possible"); | ||
364 | |||
365 | goto exit_vf_table; | ||
366 | } | ||
367 | } | ||
368 | |||
369 | /* Calculate voltages */ | ||
370 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
371 | &alt_gpc2clk, &gpc2clk_voltuv, | ||
372 | CTRL_VOLT_DOMAIN_LOGIC); | ||
373 | if (status < 0) { | ||
374 | nvgpu_err(g, "failed to get GPC2CLK LOGIC voltage"); | ||
375 | goto exit_vf_table; | ||
376 | } | ||
377 | |||
378 | status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK, | ||
379 | &alt_gpc2clk, | ||
380 | &gpc2clk_voltuv_sram, | ||
381 | CTRL_VOLT_DOMAIN_SRAM); | ||
382 | if (status < 0) { | ||
383 | nvgpu_err(g, "failed to get GPC2CLK SRAM voltage"); | ||
384 | goto exit_vf_table; | ||
385 | } | ||
386 | |||
387 | table->gpc2clk_points[i].uvolt = gpc2clk_voltuv; | ||
388 | table->gpc2clk_points[i].uvolt_sram = gpc2clk_voltuv_sram; | ||
389 | } | ||
390 | |||
391 | /* make table visible when all data has resolved in the tables */ | ||
392 | nvgpu_smp_wmb(); | ||
393 | xchg(&arb->current_vf_table, table); | ||
394 | |||
395 | exit_vf_table: | ||
396 | |||
397 | if (status < 0) | ||
398 | nvgpu_clk_arb_set_global_alarm(g, | ||
399 | EVENT(ALARM_VF_TABLE_UPDATE_FAILED)); | ||
400 | nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item); | ||
401 | |||
402 | return status; | ||
403 | } | ||
404 | |||
405 | |||
406 | static void nvgpu_clk_arb_run_vf_table_cb(struct nvgpu_clk_arb *arb) | ||
407 | { | ||
408 | struct gk20a *g = arb->g; | ||
409 | u32 err; | ||
410 | |||
411 | /* get latest vf curve from pmu */ | ||
412 | err = clk_vf_point_cache(g); | ||
413 | if (err) { | ||
414 | nvgpu_err(g, "failed to cache VF table"); | ||
415 | nvgpu_clk_arb_set_global_alarm(g, | ||
416 | EVENT(ALARM_VF_TABLE_UPDATE_FAILED)); | ||
417 | nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item); | ||
418 | |||
419 | return; | ||
420 | } | ||
421 | nvgpu_clk_arb_update_vf_table(arb); | ||
422 | } | ||
423 | |||
424 | static u8 nvgpu_clk_arb_find_vf_point(struct nvgpu_clk_arb *arb, | ||
425 | u16 *gpc2clk, u16 *sys2clk, u16 *xbar2clk, u16 *mclk, | ||
426 | u32 *voltuv, u32 *voltuv_sram, u32 *nuvmin, u32 *nuvmin_sram) | ||
427 | { | ||
428 | u16 gpc2clk_target, mclk_target; | ||
429 | u32 gpc2clk_voltuv, gpc2clk_voltuv_sram; | ||
430 | u32 mclk_voltuv, mclk_voltuv_sram; | ||
431 | u32 pstate = VF_POINT_INVALID_PSTATE; | ||
432 | struct nvgpu_clk_vf_table *table; | ||
433 | u32 index, index_mclk; | ||
434 | struct nvgpu_clk_vf_point *mclk_vf = NULL; | ||
435 | |||
436 | do { | ||
437 | gpc2clk_target = *gpc2clk; | ||
438 | mclk_target = *mclk; | ||
439 | gpc2clk_voltuv = 0; | ||
440 | gpc2clk_voltuv_sram = 0; | ||
441 | mclk_voltuv = 0; | ||
442 | mclk_voltuv_sram = 0; | ||
443 | |||
444 | table = NV_ACCESS_ONCE(arb->current_vf_table); | ||
445 | /* pointer to table can be updated by callback */ | ||
446 | nvgpu_smp_rmb(); | ||
447 | |||
448 | if (!table) | ||
449 | continue; | ||
450 | if ((!table->gpc2clk_num_points) || (!table->mclk_num_points)) { | ||
451 | nvgpu_err(arb->g, "found empty table"); | ||
452 | goto find_exit; | ||
453 | } | ||
454 | /* First we check MCLK to find out which PSTATE we are | ||
455 | * are requesting, and from there try to find the minimum | ||
456 | * GPC2CLK on the same PSTATE that satisfies the request. | ||
457 | * If no GPC2CLK can be found, then we need to up the PSTATE | ||
458 | */ | ||
459 | |||
460 | recalculate_vf_point: | ||
461 | for (index = 0; index < table->mclk_num_points; index++) { | ||
462 | if (table->mclk_points[index].mem_mhz >= mclk_target) { | ||
463 | mclk_vf = &table->mclk_points[index]; | ||
464 | break; | ||
465 | } | ||
466 | } | ||
467 | if (index == table->mclk_num_points) { | ||
468 | mclk_vf = &table->mclk_points[index-1]; | ||
469 | index = table->mclk_num_points - 1; | ||
470 | } | ||
471 | index_mclk = index; | ||
472 | |||
473 | /* round up the freq requests */ | ||
474 | for (index = 0; index < table->gpc2clk_num_points; index++) { | ||
475 | pstate = VF_POINT_COMMON_PSTATE( | ||
476 | &table->gpc2clk_points[index], mclk_vf); | ||
477 | |||
478 | if ((table->gpc2clk_points[index].gpc_mhz >= | ||
479 | gpc2clk_target) && | ||
480 | (pstate != VF_POINT_INVALID_PSTATE)) { | ||
481 | gpc2clk_target = | ||
482 | table->gpc2clk_points[index].gpc_mhz; | ||
483 | *sys2clk = | ||
484 | table->gpc2clk_points[index].sys_mhz; | ||
485 | *xbar2clk = | ||
486 | table->gpc2clk_points[index].xbar_mhz; | ||
487 | |||
488 | gpc2clk_voltuv = | ||
489 | table->gpc2clk_points[index].uvolt; | ||
490 | gpc2clk_voltuv_sram = | ||
491 | table->gpc2clk_points[index].uvolt_sram; | ||
492 | break; | ||
493 | } | ||
494 | } | ||
495 | |||
496 | if (index == table->gpc2clk_num_points) { | ||
497 | pstate = VF_POINT_COMMON_PSTATE( | ||
498 | &table->gpc2clk_points[index-1], mclk_vf); | ||
499 | if (pstate != VF_POINT_INVALID_PSTATE) { | ||
500 | gpc2clk_target = | ||
501 | table->gpc2clk_points[index-1].gpc_mhz; | ||
502 | *sys2clk = | ||
503 | table->gpc2clk_points[index-1].sys_mhz; | ||
504 | *xbar2clk = | ||
505 | table->gpc2clk_points[index-1].xbar_mhz; | ||
506 | |||
507 | gpc2clk_voltuv = | ||
508 | table->gpc2clk_points[index-1].uvolt; | ||
509 | gpc2clk_voltuv_sram = | ||
510 | table->gpc2clk_points[index-1]. | ||
511 | uvolt_sram; | ||
512 | } else if (index_mclk >= table->mclk_num_points - 1) { | ||
513 | /* There is no available combination of MCLK | ||
514 | * and GPC2CLK, we need to fail this | ||
515 | */ | ||
516 | gpc2clk_target = 0; | ||
517 | mclk_target = 0; | ||
518 | pstate = VF_POINT_INVALID_PSTATE; | ||
519 | goto find_exit; | ||
520 | } else { | ||
521 | /* recalculate with higher PSTATE */ | ||
522 | gpc2clk_target = *gpc2clk; | ||
523 | mclk_target = table->mclk_points[index_mclk+1]. | ||
524 | mem_mhz; | ||
525 | goto recalculate_vf_point; | ||
526 | } | ||
527 | } | ||
528 | |||
529 | mclk_target = mclk_vf->mem_mhz; | ||
530 | mclk_voltuv = mclk_vf->uvolt; | ||
531 | mclk_voltuv_sram = mclk_vf->uvolt_sram; | ||
532 | |||
533 | } while (!table || | ||
534 | (NV_ACCESS_ONCE(arb->current_vf_table) != table)); | ||
535 | |||
536 | find_exit: | ||
537 | *voltuv = gpc2clk_voltuv > mclk_voltuv ? gpc2clk_voltuv : mclk_voltuv; | ||
538 | *voltuv_sram = gpc2clk_voltuv_sram > mclk_voltuv_sram ? | ||
539 | gpc2clk_voltuv_sram : mclk_voltuv_sram; | ||
540 | /* noise unaware vmin */ | ||
541 | *nuvmin = mclk_voltuv; | ||
542 | *nuvmin_sram = mclk_voltuv_sram; | ||
543 | *gpc2clk = gpc2clk_target < *gpc2clk ? gpc2clk_target : *gpc2clk; | ||
544 | *mclk = mclk_target; | ||
545 | return pstate; | ||
546 | } | ||
547 | |||
548 | static int nvgpu_clk_arb_change_vf_point(struct gk20a *g, u16 gpc2clk_target, | ||
549 | u16 sys2clk_target, u16 xbar2clk_target, u16 mclk_target, u32 voltuv, | ||
550 | u32 voltuv_sram) | ||
551 | { | ||
552 | struct set_fll_clk fllclk; | ||
553 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
554 | int status; | ||
555 | |||
556 | fllclk.gpc2clkmhz = gpc2clk_target; | ||
557 | fllclk.sys2clkmhz = sys2clk_target; | ||
558 | fllclk.xbar2clkmhz = xbar2clk_target; | ||
559 | |||
560 | fllclk.voltuv = voltuv; | ||
561 | |||
562 | /* if voltage ascends we do: | ||
563 | * (1) FLL change | ||
564 | * (2) Voltage change | ||
565 | * (3) MCLK change | ||
566 | * If it goes down | ||
567 | * (1) MCLK change | ||
568 | * (2) Voltage change | ||
569 | * (3) FLL change | ||
570 | */ | ||
571 | |||
572 | /* descending */ | ||
573 | if (voltuv < arb->voltuv_actual) { | ||
574 | status = g->ops.clk.mclk_change(g, mclk_target); | ||
575 | if (status < 0) | ||
576 | return status; | ||
577 | |||
578 | status = volt_set_voltage(g, voltuv, voltuv_sram); | ||
579 | if (status < 0) | ||
580 | return status; | ||
581 | |||
582 | status = clk_set_fll_clks(g, &fllclk); | ||
583 | if (status < 0) | ||
584 | return status; | ||
585 | } else { | ||
586 | status = clk_set_fll_clks(g, &fllclk); | ||
587 | if (status < 0) | ||
588 | return status; | ||
589 | |||
590 | status = volt_set_voltage(g, voltuv, voltuv_sram); | ||
591 | if (status < 0) | ||
592 | return status; | ||
593 | |||
594 | status = g->ops.clk.mclk_change(g, mclk_target); | ||
595 | if (status < 0) | ||
596 | return status; | ||
597 | } | ||
598 | |||
599 | return 0; | ||
600 | } | ||
601 | |||
602 | static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, | ||
603 | struct nvgpu_clk_arb_target *target, | ||
604 | u32 alarm) { | ||
605 | |||
606 | struct nvgpu_clk_session *session = dev->session; | ||
607 | struct nvgpu_clk_arb *arb = session->g->clk_arb; | ||
608 | struct nvgpu_clk_notification *notification; | ||
609 | |||
610 | u32 queue_alarm_mask = 0; | ||
611 | u32 enabled_mask = 0; | ||
612 | u32 new_alarms_reported = 0; | ||
613 | u32 poll_mask = 0; | ||
614 | u32 tail, head; | ||
615 | u32 queue_index; | ||
616 | size_t size; | ||
617 | int index; | ||
618 | |||
619 | enabled_mask = nvgpu_atomic_read(&dev->enabled_mask); | ||
620 | size = arb->notification_queue.size; | ||
621 | |||
622 | /* queue global arbiter notifications in buffer */ | ||
623 | do { | ||
624 | tail = nvgpu_atomic_read(&arb->notification_queue.tail); | ||
625 | /* copy items to the queue */ | ||
626 | queue_index = nvgpu_atomic_read(&dev->queue.tail); | ||
627 | head = dev->arb_queue_head; | ||
628 | head = (tail - head) < arb->notification_queue.size ? | ||
629 | head : tail - arb->notification_queue.size; | ||
630 | |||
631 | for (index = head; _WRAPGTEQ(tail, index); index++) { | ||
632 | u32 alarm_detected; | ||
633 | |||
634 | notification = &arb->notification_queue. | ||
635 | notifications[(index+1) % size]; | ||
636 | alarm_detected = | ||
637 | NV_ACCESS_ONCE(notification->notification); | ||
638 | |||
639 | if (!(enabled_mask & alarm_detected)) | ||
640 | continue; | ||
641 | |||
642 | queue_index++; | ||
643 | dev->queue.notifications[ | ||
644 | queue_index % dev->queue.size].timestamp = | ||
645 | NV_ACCESS_ONCE(notification->timestamp); | ||
646 | |||
647 | dev->queue.notifications[ | ||
648 | queue_index % dev->queue.size].notification = | ||
649 | alarm_detected; | ||
650 | |||
651 | queue_alarm_mask |= alarm_detected; | ||
652 | } | ||
653 | } while (unlikely(nvgpu_atomic_read(&arb->notification_queue.tail) != | ||
654 | (int)tail)); | ||
655 | |||
656 | nvgpu_atomic_set(&dev->queue.tail, queue_index); | ||
657 | /* update the last notification we processed from global queue */ | ||
658 | |||
659 | dev->arb_queue_head = tail; | ||
660 | |||
661 | /* Check if current session targets are met */ | ||
662 | if (enabled_mask & EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)) { | ||
663 | if ((target->gpc2clk < session->target->gpc2clk) | ||
664 | || (target->mclk < session->target->mclk)) { | ||
665 | |||
666 | poll_mask |= (NVGPU_POLLIN | NVGPU_POLLPRI); | ||
667 | nvgpu_clk_arb_queue_notification(arb->g, &dev->queue, | ||
668 | EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)); | ||
669 | } | ||
670 | } | ||
671 | |||
672 | /* Check if there is a new VF update */ | ||
673 | if (queue_alarm_mask & EVENT(VF_UPDATE)) | ||
674 | poll_mask |= (NVGPU_POLLIN | NVGPU_POLLRDNORM); | ||
675 | |||
676 | /* Notify sticky alarms that were not reported on previous run*/ | ||
677 | new_alarms_reported = (queue_alarm_mask | | ||
678 | (alarm & ~dev->alarms_reported & queue_alarm_mask)); | ||
679 | |||
680 | if (new_alarms_reported & ~LOCAL_ALARM_MASK) { | ||
681 | /* check that we are not re-reporting */ | ||
682 | if (new_alarms_reported & EVENT(ALARM_GPU_LOST)) | ||
683 | poll_mask |= NVGPU_POLLHUP; | ||
684 | |||
685 | poll_mask |= (NVGPU_POLLIN | NVGPU_POLLPRI); | ||
686 | /* On next run do not report global alarms that were already | ||
687 | * reported, but report SHUTDOWN always | ||
688 | */ | ||
689 | dev->alarms_reported = new_alarms_reported & ~LOCAL_ALARM_MASK & | ||
690 | ~EVENT(ALARM_GPU_LOST); | ||
691 | } | ||
692 | |||
693 | if (poll_mask) { | ||
694 | nvgpu_atomic_set(&dev->poll_mask, poll_mask); | ||
695 | nvgpu_cond_broadcast_interruptible(&dev->readout_wq); | ||
696 | } | ||
697 | |||
698 | return new_alarms_reported; | ||
699 | } | ||
700 | |||
701 | static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm) | ||
702 | { | ||
703 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
704 | |||
705 | u64 current_mask; | ||
706 | u32 refcnt; | ||
707 | u32 alarm_mask; | ||
708 | u64 new_mask; | ||
709 | |||
710 | do { | ||
711 | current_mask = nvgpu_atomic64_read(&arb->alarm_mask); | ||
712 | /* atomic operations are strong so they do not need masks */ | ||
713 | |||
714 | refcnt = ((u32) (current_mask >> 32)) + 1; | ||
715 | alarm_mask = (u32) (current_mask & ~alarm); | ||
716 | new_mask = ((u64) refcnt << 32) | alarm_mask; | ||
717 | |||
718 | } while (unlikely(current_mask != | ||
719 | (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask, | ||
720 | current_mask, new_mask))); | ||
721 | } | ||
722 | |||
723 | static void nvgpu_clk_arb_run_arbiter_cb(struct nvgpu_clk_arb *arb) | ||
724 | { | ||
725 | struct nvgpu_clk_session *session; | ||
726 | struct nvgpu_clk_dev *dev; | ||
727 | struct nvgpu_clk_dev *tmp; | ||
728 | struct nvgpu_clk_arb_target *target, *actual; | ||
729 | struct gk20a *g = arb->g; | ||
730 | |||
731 | u32 pstate = VF_POINT_INVALID_PSTATE; | ||
732 | u32 voltuv, voltuv_sram; | ||
733 | bool mclk_set, gpc2clk_set; | ||
734 | u32 nuvmin, nuvmin_sram; | ||
735 | |||
736 | u32 alarms_notified = 0; | ||
737 | u32 current_alarm; | ||
738 | int status = 0; | ||
739 | |||
740 | /* Temporary variables for checking target frequency */ | ||
741 | u16 gpc2clk_target, sys2clk_target, xbar2clk_target, mclk_target; | ||
742 | u16 gpc2clk_session_target, mclk_session_target; | ||
743 | |||
744 | #ifdef CONFIG_DEBUG_FS | ||
745 | u64 t0, t1; | ||
746 | struct nvgpu_clk_arb_debug *debug; | ||
747 | |||
748 | #endif | ||
749 | |||
750 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_clk_arb, " "); | ||
751 | |||
752 | /* bail out if gpu is down */ | ||
753 | if (nvgpu_atomic64_read(&arb->alarm_mask) & EVENT(ALARM_GPU_LOST)) | ||
754 | goto exit_arb; | ||
755 | |||
756 | #ifdef CONFIG_DEBUG_FS | ||
757 | g->ops.bus.read_ptimer(g, &t0); | ||
758 | #endif | ||
759 | |||
760 | /* Only one arbiter should be running */ | ||
761 | gpc2clk_target = 0; | ||
762 | mclk_target = 0; | ||
763 | |||
764 | nvgpu_spinlock_acquire(&arb->sessions_lock); | ||
765 | nvgpu_list_for_each_entry(session, &arb->sessions, | ||
766 | nvgpu_clk_session, link) { | ||
767 | if (!session->zombie) { | ||
768 | mclk_set = false; | ||
769 | gpc2clk_set = false; | ||
770 | target = (session->target == &session->target_pool[0] ? | ||
771 | &session->target_pool[1] : | ||
772 | &session->target_pool[0]); | ||
773 | nvgpu_spinlock_acquire(&session->session_lock); | ||
774 | if (!nvgpu_list_empty(&session->targets)) { | ||
775 | /* Copy over state */ | ||
776 | target->mclk = session->target->mclk; | ||
777 | target->gpc2clk = session->target->gpc2clk; | ||
778 | /* Query the latest committed request */ | ||
779 | nvgpu_list_for_each_entry_safe(dev, tmp, &session->targets, | ||
780 | nvgpu_clk_dev, node) { | ||
781 | if (!mclk_set && dev->mclk_target_mhz) { | ||
782 | target->mclk = | ||
783 | dev->mclk_target_mhz; | ||
784 | mclk_set = true; | ||
785 | } | ||
786 | if (!gpc2clk_set && | ||
787 | dev->gpc2clk_target_mhz) { | ||
788 | target->gpc2clk = | ||
789 | dev->gpc2clk_target_mhz; | ||
790 | gpc2clk_set = true; | ||
791 | } | ||
792 | nvgpu_ref_get(&dev->refcount); | ||
793 | nvgpu_list_del(&dev->node); | ||
794 | nvgpu_spinlock_acquire(&arb->requests_lock); | ||
795 | nvgpu_list_add(&dev->node, &arb->requests); | ||
796 | nvgpu_spinlock_release(&arb->requests_lock); | ||
797 | } | ||
798 | xchg(&session->target, target); | ||
799 | } | ||
800 | nvgpu_spinlock_release(&session->session_lock); | ||
801 | |||
802 | mclk_target = mclk_target > session->target->mclk ? | ||
803 | mclk_target : session->target->mclk; | ||
804 | |||
805 | gpc2clk_target = | ||
806 | gpc2clk_target > session->target->gpc2clk ? | ||
807 | gpc2clk_target : session->target->gpc2clk; | ||
808 | } | ||
809 | } | ||
810 | nvgpu_spinlock_release(&arb->sessions_lock); | ||
811 | |||
812 | gpc2clk_target = (gpc2clk_target > 0) ? gpc2clk_target : | ||
813 | arb->gpc2clk_default_mhz; | ||
814 | |||
815 | if (gpc2clk_target < arb->gpc2clk_min) | ||
816 | gpc2clk_target = arb->gpc2clk_min; | ||
817 | |||
818 | if (gpc2clk_target > arb->gpc2clk_max) | ||
819 | gpc2clk_target = arb->gpc2clk_max; | ||
820 | |||
821 | mclk_target = (mclk_target > 0) ? mclk_target : | ||
822 | arb->mclk_default_mhz; | ||
823 | |||
824 | if (mclk_target < arb->mclk_min) | ||
825 | mclk_target = arb->mclk_min; | ||
826 | |||
827 | if (mclk_target > arb->mclk_max) | ||
828 | mclk_target = arb->mclk_max; | ||
829 | |||
830 | sys2clk_target = 0; | ||
831 | xbar2clk_target = 0; | ||
832 | |||
833 | gpc2clk_session_target = gpc2clk_target; | ||
834 | mclk_session_target = mclk_target; | ||
835 | |||
836 | /* Query the table for the closest vf point to program */ | ||
837 | pstate = nvgpu_clk_arb_find_vf_point(arb, &gpc2clk_target, | ||
838 | &sys2clk_target, &xbar2clk_target, &mclk_target, &voltuv, | ||
839 | &voltuv_sram, &nuvmin, &nuvmin_sram); | ||
840 | |||
841 | if (pstate == VF_POINT_INVALID_PSTATE) { | ||
842 | arb->status = -EINVAL; | ||
843 | /* make status visible */ | ||
844 | nvgpu_smp_mb(); | ||
845 | goto exit_arb; | ||
846 | } | ||
847 | |||
848 | if ((gpc2clk_target < gpc2clk_session_target) || | ||
849 | (mclk_target < mclk_session_target)) | ||
850 | nvgpu_clk_arb_set_global_alarm(g, | ||
851 | EVENT(ALARM_TARGET_VF_NOT_POSSIBLE)); | ||
852 | |||
853 | if ((arb->actual->gpc2clk == gpc2clk_target) && | ||
854 | (arb->actual->mclk == mclk_target) && | ||
855 | (arb->voltuv_actual == voltuv)) { | ||
856 | goto exit_arb; | ||
857 | } | ||
858 | |||
859 | /* Program clocks */ | ||
860 | /* A change in both mclk of gpc2clk may require a change in voltage */ | ||
861 | |||
862 | nvgpu_mutex_acquire(&arb->pstate_lock); | ||
863 | status = nvgpu_lpwr_disable_pg(g, false); | ||
864 | |||
865 | status = clk_pmu_freq_controller_load(g, false, | ||
866 | CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL); | ||
867 | if (status < 0) { | ||
868 | arb->status = status; | ||
869 | nvgpu_mutex_release(&arb->pstate_lock); | ||
870 | |||
871 | /* make status visible */ | ||
872 | nvgpu_smp_mb(); | ||
873 | goto exit_arb; | ||
874 | } | ||
875 | status = volt_set_noiseaware_vmin(g, nuvmin, nuvmin_sram); | ||
876 | if (status < 0) { | ||
877 | arb->status = status; | ||
878 | nvgpu_mutex_release(&arb->pstate_lock); | ||
879 | |||
880 | /* make status visible */ | ||
881 | nvgpu_smp_mb(); | ||
882 | goto exit_arb; | ||
883 | } | ||
884 | |||
885 | status = nvgpu_clk_arb_change_vf_point(g, gpc2clk_target, | ||
886 | sys2clk_target, xbar2clk_target, mclk_target, voltuv, | ||
887 | voltuv_sram); | ||
888 | if (status < 0) { | ||
889 | arb->status = status; | ||
890 | nvgpu_mutex_release(&arb->pstate_lock); | ||
891 | |||
892 | /* make status visible */ | ||
893 | nvgpu_smp_mb(); | ||
894 | goto exit_arb; | ||
895 | } | ||
896 | |||
897 | status = clk_pmu_freq_controller_load(g, true, | ||
898 | CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL); | ||
899 | if (status < 0) { | ||
900 | arb->status = status; | ||
901 | nvgpu_mutex_release(&arb->pstate_lock); | ||
902 | |||
903 | /* make status visible */ | ||
904 | nvgpu_smp_mb(); | ||
905 | goto exit_arb; | ||
906 | } | ||
907 | |||
908 | status = nvgpu_lwpr_mclk_change(g, pstate); | ||
909 | if (status < 0) { | ||
910 | arb->status = status; | ||
911 | nvgpu_mutex_release(&arb->pstate_lock); | ||
912 | |||
913 | /* make status visible */ | ||
914 | nvgpu_smp_mb(); | ||
915 | goto exit_arb; | ||
916 | } | ||
917 | |||
918 | actual = NV_ACCESS_ONCE(arb->actual) == &arb->actual_pool[0] ? | ||
919 | &arb->actual_pool[1] : &arb->actual_pool[0]; | ||
920 | |||
921 | /* do not reorder this pointer */ | ||
922 | nvgpu_smp_rmb(); | ||
923 | actual->gpc2clk = gpc2clk_target; | ||
924 | actual->mclk = mclk_target; | ||
925 | arb->voltuv_actual = voltuv; | ||
926 | actual->pstate = pstate; | ||
927 | arb->status = status; | ||
928 | |||
929 | /* Make changes visible to other threads */ | ||
930 | nvgpu_smp_wmb(); | ||
931 | xchg(&arb->actual, actual); | ||
932 | |||
933 | status = nvgpu_lpwr_enable_pg(g, false); | ||
934 | if (status < 0) { | ||
935 | arb->status = status; | ||
936 | nvgpu_mutex_release(&arb->pstate_lock); | ||
937 | |||
938 | /* make status visible */ | ||
939 | nvgpu_smp_mb(); | ||
940 | goto exit_arb; | ||
941 | } | ||
942 | |||
943 | /* status must be visible before atomic inc */ | ||
944 | nvgpu_smp_wmb(); | ||
945 | nvgpu_atomic_inc(&arb->req_nr); | ||
946 | |||
947 | /* Unlock pstate change for PG */ | ||
948 | nvgpu_mutex_release(&arb->pstate_lock); | ||
949 | |||
950 | /* VF Update complete */ | ||
951 | nvgpu_clk_arb_set_global_alarm(g, EVENT(VF_UPDATE)); | ||
952 | |||
953 | nvgpu_cond_signal_interruptible(&arb->request_wq); | ||
954 | |||
955 | #ifdef CONFIG_DEBUG_FS | ||
956 | g->ops.bus.read_ptimer(g, &t1); | ||
957 | |||
958 | debug = arb->debug == &arb->debug_pool[0] ? | ||
959 | &arb->debug_pool[1] : &arb->debug_pool[0]; | ||
960 | |||
961 | memcpy(debug, arb->debug, sizeof(arb->debug_pool[0])); | ||
962 | debug->switch_num++; | ||
963 | |||
964 | if (debug->switch_num == 1) { | ||
965 | debug->switch_max = debug->switch_min = | ||
966 | debug->switch_avg = (t1-t0)/1000; | ||
967 | debug->switch_std = 0; | ||
968 | } else { | ||
969 | s64 prev_avg; | ||
970 | s64 curr = (t1-t0)/1000; | ||
971 | |||
972 | debug->switch_max = curr > debug->switch_max ? | ||
973 | curr : debug->switch_max; | ||
974 | debug->switch_min = debug->switch_min ? | ||
975 | (curr < debug->switch_min ? | ||
976 | curr : debug->switch_min) : curr; | ||
977 | prev_avg = debug->switch_avg; | ||
978 | debug->switch_avg = (curr + | ||
979 | (debug->switch_avg * (debug->switch_num-1))) / | ||
980 | debug->switch_num; | ||
981 | debug->switch_std += | ||
982 | (curr - debug->switch_avg) * (curr - prev_avg); | ||
983 | } | ||
984 | /* commit changes before exchanging debug pointer */ | ||
985 | nvgpu_smp_wmb(); | ||
986 | xchg(&arb->debug, debug); | ||
987 | #endif | ||
988 | |||
989 | exit_arb: | ||
990 | if (status < 0) { | ||
991 | nvgpu_err(g, "Error in arbiter update"); | ||
992 | nvgpu_clk_arb_set_global_alarm(g, | ||
993 | EVENT(ALARM_CLOCK_ARBITER_FAILED)); | ||
994 | } | ||
995 | |||
996 | current_alarm = (u32) nvgpu_atomic64_read(&arb->alarm_mask); | ||
997 | /* notify completion for all requests */ | ||
998 | nvgpu_spinlock_acquire(&arb->requests_lock); | ||
999 | nvgpu_list_for_each_entry_safe(dev, tmp, &arb->requests, | ||
1000 | nvgpu_clk_dev, node) { | ||
1001 | nvgpu_atomic_set(&dev->poll_mask, NVGPU_POLLIN | NVGPU_POLLRDNORM); | ||
1002 | nvgpu_cond_signal_interruptible(&dev->readout_wq); | ||
1003 | nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | ||
1004 | nvgpu_list_del(&dev->node); | ||
1005 | } | ||
1006 | nvgpu_spinlock_release(&arb->requests_lock); | ||
1007 | |||
1008 | nvgpu_atomic_set(&arb->notification_queue.head, | ||
1009 | nvgpu_atomic_read(&arb->notification_queue.tail)); | ||
1010 | /* notify event for all users */ | ||
1011 | nvgpu_spinlock_acquire(&arb->users_lock); | ||
1012 | nvgpu_list_for_each_entry(dev, &arb->users, nvgpu_clk_dev, link) { | ||
1013 | alarms_notified |= | ||
1014 | nvgpu_clk_arb_notify(dev, arb->actual, current_alarm); | ||
1015 | } | ||
1016 | nvgpu_spinlock_release(&arb->users_lock); | ||
1017 | |||
1018 | /* clear alarms */ | ||
1019 | nvgpu_clk_arb_clear_global_alarm(g, alarms_notified & | ||
1020 | ~EVENT(ALARM_GPU_LOST)); | ||
1021 | } | ||
1022 | |||
1023 | /* | ||
1024 | * Process one scheduled work item. | ||
1025 | */ | ||
1026 | static void nvgpu_clk_arb_worker_process_item( | ||
1027 | struct nvgpu_clk_arb_work_item *work_item) | ||
1028 | { | ||
1029 | nvgpu_log(work_item->arb->g, gpu_dbg_fn | gpu_dbg_clk_arb, " "); | ||
1030 | |||
1031 | if (work_item->item_type == CLK_ARB_WORK_UPDATE_VF_TABLE) | ||
1032 | nvgpu_clk_arb_run_vf_table_cb(work_item->arb); | ||
1033 | else if (work_item->item_type == CLK_ARB_WORK_UPDATE_ARB) | ||
1034 | nvgpu_clk_arb_run_arbiter_cb(work_item->arb); | ||
1035 | } | ||
1036 | |||
1037 | /** | ||
1038 | * Tell the worker that one more work needs to be done. | ||
1039 | * | ||
1040 | * Increase the work counter to synchronize the worker with the new work. Wake | ||
1041 | * up the worker. If the worker was already running, it will handle this work | ||
1042 | * before going to sleep. | ||
1043 | */ | ||
1044 | static int nvgpu_clk_arb_worker_wakeup(struct gk20a *g) | ||
1045 | { | ||
1046 | int put; | ||
1047 | |||
1048 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_clk_arb, " "); | ||
1049 | |||
1050 | put = nvgpu_atomic_inc_return(&g->clk_arb_worker.put); | ||
1051 | nvgpu_cond_signal_interruptible(&g->clk_arb_worker.wq); | ||
1052 | |||
1053 | return put; | ||
1054 | } | ||
1055 | |||
1056 | /** | ||
1057 | * Test if there is some work pending. | ||
1058 | * | ||
1059 | * This is a pair for nvgpu_clk_arb_worker_wakeup to be called from the | ||
1060 | * worker. The worker has an internal work counter which is incremented once | ||
1061 | * per finished work item. This is compared with the number of queued jobs. | ||
1062 | */ | ||
1063 | static bool nvgpu_clk_arb_worker_pending(struct gk20a *g, int get) | ||
1064 | { | ||
1065 | bool pending = nvgpu_atomic_read(&g->clk_arb_worker.put) != get; | ||
1066 | |||
1067 | /* We don't need barriers because they are implicit in locking */ | ||
1068 | return pending; | ||
1069 | } | ||
1070 | |||
1071 | /** | ||
1072 | * Process the queued works for the worker thread serially. | ||
1073 | * | ||
1074 | * Flush all the work items in the queue one by one. This may block timeout | ||
1075 | * handling for a short while, as these are serialized. | ||
1076 | */ | ||
1077 | static void nvgpu_clk_arb_worker_process(struct gk20a *g, int *get) | ||
1078 | { | ||
1079 | |||
1080 | while (nvgpu_clk_arb_worker_pending(g, *get)) { | ||
1081 | struct nvgpu_clk_arb_work_item *work_item = NULL; | ||
1082 | |||
1083 | nvgpu_spinlock_acquire(&g->clk_arb_worker.items_lock); | ||
1084 | if (!nvgpu_list_empty(&g->clk_arb_worker.items)) { | ||
1085 | work_item = nvgpu_list_first_entry(&g->clk_arb_worker.items, | ||
1086 | nvgpu_clk_arb_work_item, worker_item); | ||
1087 | nvgpu_list_del(&work_item->worker_item); | ||
1088 | } | ||
1089 | nvgpu_spinlock_release(&g->clk_arb_worker.items_lock); | ||
1090 | |||
1091 | if (!work_item) { | ||
1092 | /* | ||
1093 | * Woke up for some other reason, but there are no | ||
1094 | * other reasons than a work item added in the items list | ||
1095 | * currently, so warn and ack the message. | ||
1096 | */ | ||
1097 | nvgpu_warn(g, "Spurious worker event!"); | ||
1098 | ++*get; | ||
1099 | break; | ||
1100 | } | ||
1101 | |||
1102 | nvgpu_clk_arb_worker_process_item(work_item); | ||
1103 | ++*get; | ||
1104 | } | ||
1105 | } | ||
1106 | |||
1107 | /* | ||
1108 | * Process all work items found in the clk arbiter work queue. | ||
1109 | */ | ||
1110 | static int nvgpu_clk_arb_poll_worker(void *arg) | ||
1111 | { | ||
1112 | struct gk20a *g = (struct gk20a *)arg; | ||
1113 | struct gk20a_worker *worker = &g->clk_arb_worker; | ||
1114 | int get = 0; | ||
1115 | |||
1116 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_clk_arb, " "); | ||
1117 | |||
1118 | while (!nvgpu_thread_should_stop(&worker->poll_task)) { | ||
1119 | int ret; | ||
1120 | |||
1121 | ret = NVGPU_COND_WAIT_INTERRUPTIBLE( | ||
1122 | &worker->wq, | ||
1123 | nvgpu_clk_arb_worker_pending(g, get), 0); | ||
1124 | |||
1125 | if (ret == 0) | ||
1126 | nvgpu_clk_arb_worker_process(g, &get); | ||
1127 | } | ||
1128 | return 0; | ||
1129 | } | ||
1130 | |||
1131 | static int __nvgpu_clk_arb_worker_start(struct gk20a *g) | ||
1132 | { | ||
1133 | char thread_name[64]; | ||
1134 | int err = 0; | ||
1135 | |||
1136 | if (nvgpu_thread_is_running(&g->clk_arb_worker.poll_task)) | ||
1137 | return err; | ||
1138 | |||
1139 | nvgpu_mutex_acquire(&g->clk_arb_worker.start_lock); | ||
1140 | |||
1141 | /* | ||
1142 | * Mutexes have implicit barriers, so there is no risk of a thread | ||
1143 | * having a stale copy of the poll_task variable as the call to | ||
1144 | * thread_is_running is volatile | ||
1145 | */ | ||
1146 | |||
1147 | if (nvgpu_thread_is_running(&g->clk_arb_worker.poll_task)) { | ||
1148 | nvgpu_mutex_release(&g->clk_arb_worker.start_lock); | ||
1149 | return err; | ||
1150 | } | ||
1151 | |||
1152 | snprintf(thread_name, sizeof(thread_name), | ||
1153 | "nvgpu_clk_arb_poll_%s", g->name); | ||
1154 | |||
1155 | err = nvgpu_thread_create(&g->clk_arb_worker.poll_task, g, | ||
1156 | nvgpu_clk_arb_poll_worker, thread_name); | ||
1157 | |||
1158 | nvgpu_mutex_release(&g->clk_arb_worker.start_lock); | ||
1159 | return err; | ||
1160 | } | ||
1161 | |||
1162 | /** | ||
1163 | * Append a work item to the worker's list. | ||
1164 | * | ||
1165 | * This adds work item to the end of the list and wakes the worker | ||
1166 | * up immediately. If the work item already existed in the list, it's not added, | ||
1167 | * because in that case it has been scheduled already but has not yet been | ||
1168 | * processed. | ||
1169 | */ | ||
1170 | void nvgpu_clk_arb_worker_enqueue(struct gk20a *g, | ||
1171 | struct nvgpu_clk_arb_work_item *work_item) | ||
1172 | { | ||
1173 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_clk_arb, " "); | ||
1174 | |||
1175 | /* | ||
1176 | * Warn if worker thread cannot run | ||
1177 | */ | ||
1178 | if (WARN_ON(__nvgpu_clk_arb_worker_start(g))) { | ||
1179 | nvgpu_warn(g, "clk arb worker cannot run!"); | ||
1180 | return; | ||
1181 | } | ||
1182 | |||
1183 | nvgpu_spinlock_acquire(&g->clk_arb_worker.items_lock); | ||
1184 | if (!nvgpu_list_empty(&work_item->worker_item)) { | ||
1185 | /* | ||
1186 | * Already queued, so will get processed eventually. | ||
1187 | * The worker is probably awake already. | ||
1188 | */ | ||
1189 | nvgpu_spinlock_release(&g->clk_arb_worker.items_lock); | ||
1190 | return; | ||
1191 | } | ||
1192 | nvgpu_list_add_tail(&work_item->worker_item, &g->clk_arb_worker.items); | ||
1193 | nvgpu_spinlock_release(&g->clk_arb_worker.items_lock); | ||
1194 | |||
1195 | nvgpu_clk_arb_worker_wakeup(g); | ||
1196 | } | ||
1197 | |||
1198 | /** | ||
1199 | * Initialize the clk arb worker's metadata and start the background thread. | ||
1200 | */ | ||
1201 | int nvgpu_clk_arb_worker_init(struct gk20a *g) | ||
1202 | { | ||
1203 | int err; | ||
1204 | |||
1205 | nvgpu_atomic_set(&g->clk_arb_worker.put, 0); | ||
1206 | nvgpu_cond_init(&g->clk_arb_worker.wq); | ||
1207 | nvgpu_init_list_node(&g->clk_arb_worker.items); | ||
1208 | nvgpu_spinlock_init(&g->clk_arb_worker.items_lock); | ||
1209 | err = nvgpu_mutex_init(&g->clk_arb_worker.start_lock); | ||
1210 | if (err) | ||
1211 | goto error_check; | ||
1212 | |||
1213 | err = __nvgpu_clk_arb_worker_start(g); | ||
1214 | error_check: | ||
1215 | if (err) { | ||
1216 | nvgpu_err(g, "failed to start clk arb poller thread"); | ||
1217 | return err; | ||
1218 | } | ||
1219 | return 0; | ||
1220 | } | ||
1221 | |||
1222 | int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | ||
1223 | { | ||
1224 | struct nvgpu_clk_arb *arb; | ||
1225 | u16 default_mhz; | ||
1226 | int err; | ||
1227 | int index; | ||
1228 | struct nvgpu_clk_vf_table *table; | ||
1229 | |||
1230 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_clk_arb, " "); | ||
1231 | |||
1232 | if (!g->ops.clk_arb.get_arbiter_clk_domains) | ||
1233 | return 0; | ||
1234 | |||
1235 | arb = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_arb)); | ||
1236 | if (!arb) | ||
1237 | return -ENOMEM; | ||
1238 | |||
1239 | err = nvgpu_mutex_init(&arb->pstate_lock); | ||
1240 | if (err) | ||
1241 | goto mutex_fail; | ||
1242 | nvgpu_spinlock_init(&arb->sessions_lock); | ||
1243 | nvgpu_spinlock_init(&arb->users_lock); | ||
1244 | nvgpu_spinlock_init(&arb->requests_lock); | ||
1245 | |||
1246 | arb->mclk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16)); | ||
1247 | if (!arb->mclk_f_points) { | ||
1248 | err = -ENOMEM; | ||
1249 | goto init_fail; | ||
1250 | } | ||
1251 | |||
1252 | arb->gpc2clk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16)); | ||
1253 | if (!arb->gpc2clk_f_points) { | ||
1254 | err = -ENOMEM; | ||
1255 | goto init_fail; | ||
1256 | } | ||
1257 | |||
1258 | for (index = 0; index < 2; index++) { | ||
1259 | table = &arb->vf_table_pool[index]; | ||
1260 | table->gpc2clk_num_points = MAX_F_POINTS; | ||
1261 | table->mclk_num_points = MAX_F_POINTS; | ||
1262 | |||
1263 | table->gpc2clk_points = nvgpu_kcalloc(g, MAX_F_POINTS, | ||
1264 | sizeof(struct nvgpu_clk_vf_point)); | ||
1265 | if (!table->gpc2clk_points) { | ||
1266 | err = -ENOMEM; | ||
1267 | goto init_fail; | ||
1268 | } | ||
1269 | |||
1270 | |||
1271 | table->mclk_points = nvgpu_kcalloc(g, MAX_F_POINTS, | ||
1272 | sizeof(struct nvgpu_clk_vf_point)); | ||
1273 | if (!table->mclk_points) { | ||
1274 | err = -ENOMEM; | ||
1275 | goto init_fail; | ||
1276 | } | ||
1277 | } | ||
1278 | |||
1279 | g->clk_arb = arb; | ||
1280 | arb->g = g; | ||
1281 | |||
1282 | err = g->ops.clk_arb.get_arbiter_clk_default(g, | ||
1283 | CTRL_CLK_DOMAIN_MCLK, &default_mhz); | ||
1284 | if (err < 0) { | ||
1285 | err = -EINVAL; | ||
1286 | goto init_fail; | ||
1287 | } | ||
1288 | |||
1289 | arb->mclk_default_mhz = default_mhz; | ||
1290 | |||
1291 | err = g->ops.clk_arb.get_arbiter_clk_default(g, | ||
1292 | CTRL_CLK_DOMAIN_GPC2CLK, &default_mhz); | ||
1293 | if (err < 0) { | ||
1294 | err = -EINVAL; | ||
1295 | goto init_fail; | ||
1296 | } | ||
1297 | |||
1298 | arb->gpc2clk_default_mhz = default_mhz; | ||
1299 | |||
1300 | arb->actual = &arb->actual_pool[0]; | ||
1301 | |||
1302 | nvgpu_atomic_set(&arb->req_nr, 0); | ||
1303 | |||
1304 | nvgpu_atomic64_set(&arb->alarm_mask, 0); | ||
1305 | err = nvgpu_clk_notification_queue_alloc(g, &arb->notification_queue, | ||
1306 | DEFAULT_EVENT_NUMBER); | ||
1307 | if (err < 0) | ||
1308 | goto init_fail; | ||
1309 | |||
1310 | nvgpu_init_list_node(&arb->users); | ||
1311 | nvgpu_init_list_node(&arb->sessions); | ||
1312 | nvgpu_init_list_node(&arb->requests); | ||
1313 | |||
1314 | nvgpu_cond_init(&arb->request_wq); | ||
1315 | |||
1316 | nvgpu_init_list_node(&arb->update_vf_table_work_item.worker_item); | ||
1317 | nvgpu_init_list_node(&arb->update_arb_work_item.worker_item); | ||
1318 | arb->update_vf_table_work_item.arb = arb; | ||
1319 | arb->update_arb_work_item.arb = arb; | ||
1320 | arb->update_vf_table_work_item.item_type = CLK_ARB_WORK_UPDATE_VF_TABLE; | ||
1321 | arb->update_arb_work_item.item_type = CLK_ARB_WORK_UPDATE_ARB; | ||
1322 | |||
1323 | err = nvgpu_clk_arb_worker_init(g); | ||
1324 | if (err < 0) | ||
1325 | goto init_fail; | ||
1326 | |||
1327 | #ifdef CONFIG_DEBUG_FS | ||
1328 | arb->debug = &arb->debug_pool[0]; | ||
1329 | |||
1330 | if (!arb->debugfs_set) { | ||
1331 | if (nvgpu_clk_arb_debugfs_init(g)) | ||
1332 | arb->debugfs_set = true; | ||
1333 | } | ||
1334 | #endif | ||
1335 | err = clk_vf_point_cache(g); | ||
1336 | if (err < 0) | ||
1337 | goto init_fail; | ||
1338 | |||
1339 | err = nvgpu_clk_arb_update_vf_table(arb); | ||
1340 | if (err < 0) | ||
1341 | goto init_fail; | ||
1342 | do { | ||
1343 | /* Check that first run is completed */ | ||
1344 | nvgpu_smp_mb(); | ||
1345 | NVGPU_COND_WAIT_INTERRUPTIBLE(&arb->request_wq, | ||
1346 | nvgpu_atomic_read(&arb->req_nr), 0); | ||
1347 | } while (!nvgpu_atomic_read(&arb->req_nr)); | ||
1348 | |||
1349 | |||
1350 | return arb->status; | ||
1351 | |||
1352 | init_fail: | ||
1353 | nvgpu_kfree(g, arb->gpc2clk_f_points); | ||
1354 | nvgpu_kfree(g, arb->mclk_f_points); | ||
1355 | |||
1356 | for (index = 0; index < 2; index++) { | ||
1357 | nvgpu_kfree(g, arb->vf_table_pool[index].gpc2clk_points); | ||
1358 | nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points); | ||
1359 | } | ||
1360 | |||
1361 | nvgpu_mutex_destroy(&arb->pstate_lock); | ||
1362 | |||
1363 | mutex_fail: | ||
1364 | nvgpu_kfree(g, arb); | ||
1365 | |||
1366 | return err; | ||
1367 | } | ||
1368 | |||
1369 | void nvgpu_clk_arb_send_thermal_alarm(struct gk20a *g) | ||
1370 | { | ||
1371 | nvgpu_clk_arb_schedule_alarm(g, | ||
1372 | (0x1UL << NVGPU_EVENT_ALARM_THERMAL_ABOVE_THRESHOLD)); | ||
1373 | } | ||
1374 | |||
1375 | void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm) | ||
1376 | { | ||
1377 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1378 | |||
1379 | nvgpu_clk_arb_set_global_alarm(g, alarm); | ||
1380 | nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item); | ||
1381 | } | ||
1382 | |||
1383 | void nvgpu_clk_arb_worker_deinit(struct gk20a *g) | ||
1384 | { | ||
1385 | nvgpu_mutex_acquire(&g->clk_arb_worker.start_lock); | ||
1386 | nvgpu_thread_stop(&g->clk_arb_worker.poll_task); | ||
1387 | nvgpu_mutex_release(&g->clk_arb_worker.start_lock); | ||
1388 | } | ||
1389 | |||
1390 | void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g) | ||
1391 | { | ||
1392 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1393 | int index; | ||
1394 | |||
1395 | if (arb) { | ||
1396 | nvgpu_clk_arb_worker_deinit(g); | ||
1397 | |||
1398 | nvgpu_kfree(g, arb->gpc2clk_f_points); | ||
1399 | nvgpu_kfree(g, arb->mclk_f_points); | ||
1400 | |||
1401 | for (index = 0; index < 2; index++) { | ||
1402 | nvgpu_kfree(g, | ||
1403 | arb->vf_table_pool[index].gpc2clk_points); | ||
1404 | nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points); | ||
1405 | } | ||
1406 | nvgpu_mutex_destroy(&g->clk_arb->pstate_lock); | ||
1407 | nvgpu_kfree(g, g->clk_arb); | ||
1408 | g->clk_arb = NULL; | ||
1409 | } | ||
1410 | } | ||
1411 | |||
1412 | int nvgpu_clk_arb_init_session(struct gk20a *g, | ||
1413 | struct nvgpu_clk_session **_session) | ||
1414 | { | ||
1415 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1416 | struct nvgpu_clk_session *session = *(_session); | ||
1417 | |||
1418 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_clk_arb, " "); | ||
1419 | |||
1420 | if (!g->ops.clk_arb.get_arbiter_clk_domains) | ||
1421 | return 0; | ||
1422 | |||
1423 | session = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_session)); | ||
1424 | if (!session) | ||
1425 | return -ENOMEM; | ||
1426 | session->g = g; | ||
1427 | |||
1428 | nvgpu_ref_init(&session->refcount); | ||
1429 | |||
1430 | session->zombie = false; | ||
1431 | session->target_pool[0].pstate = CTRL_PERF_PSTATE_P8; | ||
1432 | /* make sure that the initialization of the pool is visible | ||
1433 | * before the update | ||
1434 | */ | ||
1435 | nvgpu_smp_wmb(); | ||
1436 | session->target = &session->target_pool[0]; | ||
1437 | |||
1438 | nvgpu_init_list_node(&session->targets); | ||
1439 | nvgpu_spinlock_init(&session->session_lock); | ||
1440 | |||
1441 | nvgpu_spinlock_acquire(&arb->sessions_lock); | ||
1442 | nvgpu_list_add_tail(&session->link, &arb->sessions); | ||
1443 | nvgpu_spinlock_release(&arb->sessions_lock); | ||
1444 | |||
1445 | *_session = session; | ||
1446 | |||
1447 | return 0; | ||
1448 | } | ||
1449 | |||
1450 | void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount) | ||
1451 | { | ||
1452 | struct nvgpu_clk_dev *dev = container_of(refcount, | ||
1453 | struct nvgpu_clk_dev, refcount); | ||
1454 | struct nvgpu_clk_session *session = dev->session; | ||
1455 | |||
1456 | nvgpu_kfree(session->g, dev); | ||
1457 | } | ||
1458 | |||
1459 | void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount) | ||
1460 | { | ||
1461 | struct nvgpu_clk_session *session = container_of(refcount, | ||
1462 | struct nvgpu_clk_session, refcount); | ||
1463 | struct nvgpu_clk_arb *arb = session->g->clk_arb; | ||
1464 | struct gk20a *g = session->g; | ||
1465 | struct nvgpu_clk_dev *dev, *tmp; | ||
1466 | |||
1467 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_clk_arb, " "); | ||
1468 | |||
1469 | if (arb) { | ||
1470 | nvgpu_spinlock_acquire(&arb->sessions_lock); | ||
1471 | nvgpu_list_del(&session->link); | ||
1472 | nvgpu_spinlock_release(&arb->sessions_lock); | ||
1473 | } | ||
1474 | |||
1475 | nvgpu_spinlock_acquire(&session->session_lock); | ||
1476 | nvgpu_list_for_each_entry_safe(dev, tmp, &session->targets, | ||
1477 | nvgpu_clk_dev, node) { | ||
1478 | nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | ||
1479 | nvgpu_list_del(&dev->node); | ||
1480 | } | ||
1481 | nvgpu_spinlock_release(&session->session_lock); | ||
1482 | |||
1483 | nvgpu_kfree(g, session); | ||
1484 | } | ||
1485 | |||
1486 | void nvgpu_clk_arb_release_session(struct gk20a *g, | ||
1487 | struct nvgpu_clk_session *session) | ||
1488 | { | ||
1489 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1490 | |||
1491 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_clk_arb, " "); | ||
1492 | |||
1493 | session->zombie = true; | ||
1494 | nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session); | ||
1495 | if (arb) | ||
1496 | nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item); | ||
1497 | } | ||
1498 | |||
1499 | void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g) | ||
1500 | { | ||
1501 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1502 | |||
1503 | nvgpu_clk_arb_worker_enqueue(g, &arb->update_vf_table_work_item); | ||
1504 | } | ||
1505 | |||
1506 | /* This function is inherently unsafe to call while arbiter is running | ||
1507 | * arbiter must be blocked before calling this function | ||
1508 | */ | ||
1509 | int nvgpu_clk_arb_get_current_pstate(struct gk20a *g) | ||
1510 | { | ||
1511 | return NV_ACCESS_ONCE(g->clk_arb->actual->pstate); | ||
1512 | } | ||
1513 | |||
1514 | void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock) | ||
1515 | { | ||
1516 | struct nvgpu_clk_arb *arb = g->clk_arb; | ||
1517 | |||
1518 | if (lock) | ||
1519 | nvgpu_mutex_acquire(&arb->pstate_lock); | ||
1520 | else | ||
1521 | nvgpu_mutex_release(&arb->pstate_lock); | ||
1522 | } | ||
diff --git a/drivers/gpu/nvgpu/common/linux/clk_arb_linux.h b/drivers/gpu/nvgpu/common/linux/clk_arb_linux.h deleted file mode 100644 index 464590d5..00000000 --- a/drivers/gpu/nvgpu/common/linux/clk_arb_linux.h +++ /dev/null | |||
@@ -1,163 +0,0 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This software is licensed under the terms of the GNU General Public | ||
5 | * License version 2, as published by the Free Software Foundation, and | ||
6 | * may be copied, distributed, and modified under those terms. | ||
7 | * | ||
8 | * This program is distributed in the hope that it will be useful, | ||
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
11 | * GNU General Public License for more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #ifndef __NVGPU_CLK_ARB_LINUX_H__ | ||
18 | #define __NVGPU_CLK_ARB_LINUX_H__ | ||
19 | |||
20 | #include <nvgpu/types.h> | ||
21 | #include <nvgpu/bitops.h> | ||
22 | #include <nvgpu/lock.h> | ||
23 | #include <nvgpu/kmem.h> | ||
24 | #include <nvgpu/atomic.h> | ||
25 | #include <nvgpu/bug.h> | ||
26 | #include <nvgpu/kref.h> | ||
27 | #include <nvgpu/log.h> | ||
28 | #include <nvgpu/barrier.h> | ||
29 | #include <nvgpu/cond.h> | ||
30 | #include <nvgpu/list.h> | ||
31 | |||
32 | #include "gk20a/gk20a.h" | ||
33 | #include "clk/clk.h" | ||
34 | #include "pstate/pstate.h" | ||
35 | #include "lpwr/lpwr.h" | ||
36 | #include "volt/volt.h" | ||
37 | |||
38 | /* | ||
39 | * The defines here should finally move to clk_arb.h, once these are | ||
40 | * refactored to be free of Linux fields. | ||
41 | */ | ||
42 | |||
43 | enum clk_arb_work_item_type { | ||
44 | CLK_ARB_WORK_UPDATE_VF_TABLE, | ||
45 | CLK_ARB_WORK_UPDATE_ARB | ||
46 | }; | ||
47 | |||
48 | struct nvgpu_clk_arb_work_item { | ||
49 | enum clk_arb_work_item_type item_type; | ||
50 | struct nvgpu_clk_arb *arb; | ||
51 | struct nvgpu_list_node worker_item; | ||
52 | }; | ||
53 | |||
54 | struct nvgpu_clk_arb { | ||
55 | struct nvgpu_spinlock sessions_lock; | ||
56 | struct nvgpu_spinlock users_lock; | ||
57 | struct nvgpu_spinlock requests_lock; | ||
58 | |||
59 | struct nvgpu_mutex pstate_lock; | ||
60 | struct nvgpu_list_node users; | ||
61 | struct nvgpu_list_node sessions; | ||
62 | struct nvgpu_list_node requests; | ||
63 | |||
64 | struct gk20a *g; | ||
65 | int status; | ||
66 | |||
67 | struct nvgpu_clk_arb_target actual_pool[2]; | ||
68 | struct nvgpu_clk_arb_target *actual; | ||
69 | |||
70 | u16 gpc2clk_default_mhz; | ||
71 | u16 mclk_default_mhz; | ||
72 | u32 voltuv_actual; | ||
73 | |||
74 | u16 gpc2clk_min, gpc2clk_max; | ||
75 | u16 mclk_min, mclk_max; | ||
76 | |||
77 | struct nvgpu_clk_arb_work_item update_vf_table_work_item; | ||
78 | struct nvgpu_clk_arb_work_item update_arb_work_item; | ||
79 | |||
80 | struct nvgpu_cond request_wq; | ||
81 | |||
82 | struct nvgpu_clk_vf_table *current_vf_table; | ||
83 | struct nvgpu_clk_vf_table vf_table_pool[2]; | ||
84 | u32 vf_table_index; | ||
85 | |||
86 | u16 *mclk_f_points; | ||
87 | nvgpu_atomic_t req_nr; | ||
88 | |||
89 | u32 mclk_f_numpoints; | ||
90 | u16 *gpc2clk_f_points; | ||
91 | u32 gpc2clk_f_numpoints; | ||
92 | |||
93 | nvgpu_atomic64_t alarm_mask; | ||
94 | struct nvgpu_clk_notification_queue notification_queue; | ||
95 | |||
96 | #ifdef CONFIG_DEBUG_FS | ||
97 | struct nvgpu_clk_arb_debug debug_pool[2]; | ||
98 | struct nvgpu_clk_arb_debug *debug; | ||
99 | bool debugfs_set; | ||
100 | #endif | ||
101 | }; | ||
102 | |||
103 | struct nvgpu_clk_dev { | ||
104 | struct nvgpu_clk_session *session; | ||
105 | union { | ||
106 | struct nvgpu_list_node link; | ||
107 | struct nvgpu_list_node node; | ||
108 | }; | ||
109 | struct nvgpu_cond readout_wq; | ||
110 | nvgpu_atomic_t poll_mask; | ||
111 | u16 gpc2clk_target_mhz; | ||
112 | u16 mclk_target_mhz; | ||
113 | u32 alarms_reported; | ||
114 | nvgpu_atomic_t enabled_mask; | ||
115 | struct nvgpu_clk_notification_queue queue; | ||
116 | u32 arb_queue_head; | ||
117 | struct nvgpu_ref refcount; | ||
118 | }; | ||
119 | |||
120 | struct nvgpu_clk_session { | ||
121 | bool zombie; | ||
122 | struct gk20a *g; | ||
123 | struct nvgpu_ref refcount; | ||
124 | struct nvgpu_list_node link; | ||
125 | struct nvgpu_list_node targets; | ||
126 | |||
127 | struct nvgpu_spinlock session_lock; | ||
128 | struct nvgpu_clk_arb_target target_pool[2]; | ||
129 | struct nvgpu_clk_arb_target *target; | ||
130 | }; | ||
131 | |||
132 | static inline struct nvgpu_clk_session * | ||
133 | nvgpu_clk_session_from_link(struct nvgpu_list_node *node) | ||
134 | { | ||
135 | return (struct nvgpu_clk_session *) | ||
136 | ((uintptr_t)node - offsetof(struct nvgpu_clk_session, link)); | ||
137 | }; | ||
138 | |||
139 | static inline struct nvgpu_clk_dev * | ||
140 | nvgpu_clk_dev_from_node(struct nvgpu_list_node *node) | ||
141 | { | ||
142 | return (struct nvgpu_clk_dev *) | ||
143 | ((uintptr_t)node - offsetof(struct nvgpu_clk_dev, node)); | ||
144 | }; | ||
145 | |||
146 | static inline struct nvgpu_clk_dev * | ||
147 | nvgpu_clk_dev_from_link(struct nvgpu_list_node *node) | ||
148 | { | ||
149 | return (struct nvgpu_clk_dev *) | ||
150 | ((uintptr_t)node - offsetof(struct nvgpu_clk_dev, link)); | ||
151 | }; | ||
152 | |||
153 | static inline struct nvgpu_clk_arb_work_item * | ||
154 | nvgpu_clk_arb_work_item_from_worker_item(struct nvgpu_list_node *node) | ||
155 | { | ||
156 | return (struct nvgpu_clk_arb_work_item *) | ||
157 | ((uintptr_t)node - offsetof(struct nvgpu_clk_arb_work_item, worker_item)); | ||
158 | }; | ||
159 | |||
160 | void nvgpu_clk_arb_worker_enqueue(struct gk20a *g, | ||
161 | struct nvgpu_clk_arb_work_item *work_item); | ||
162 | #endif /* __NVGPU_CLK_ARB_LINUX_H__ */ | ||
163 | |||
diff --git a/drivers/gpu/nvgpu/common/linux/ioctl_clk_arb.c b/drivers/gpu/nvgpu/common/linux/ioctl_clk_arb.c index 3ab8cf9e..e4ac68a3 100644 --- a/drivers/gpu/nvgpu/common/linux/ioctl_clk_arb.c +++ b/drivers/gpu/nvgpu/common/linux/ioctl_clk_arb.c | |||
@@ -38,7 +38,6 @@ | |||
38 | 38 | ||
39 | #include "gk20a/gk20a.h" | 39 | #include "gk20a/gk20a.h" |
40 | #include "clk/clk.h" | 40 | #include "clk/clk.h" |
41 | #include "clk_arb_linux.h" | ||
42 | #include "pstate/pstate.h" | 41 | #include "pstate/pstate.h" |
43 | #include "lpwr/lpwr.h" | 42 | #include "lpwr/lpwr.h" |
44 | #include "volt/volt.h" | 43 | #include "volt/volt.h" |