1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
|
/*
* Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef __NVGPU_CLK_ARB_H__
#define __NVGPU_CLK_ARB_H__
#include <nvgpu/types.h>
#include <nvgpu/bitops.h>
#include <nvgpu/lock.h>
#include <nvgpu/kmem.h>
#include <nvgpu/atomic.h>
#include <nvgpu/bug.h>
#include <nvgpu/kref.h>
#include <nvgpu/log.h>
#include <nvgpu/barrier.h>
#include <nvgpu/cond.h>
#include "gk20a/gk20a.h"
#include "clk/clk.h"
#include "pstate/pstate.h"
#include "lpwr/lpwr.h"
#include "volt/volt.h"
#define MAX_F_POINTS 256
#define DEFAULT_EVENT_NUMBER 32
struct nvgpu_clk_dev;
struct nvgpu_clk_arb_target;
struct nvgpu_clk_notification_queue;
struct nvgpu_clk_session;
#define VF_POINT_INVALID_PSTATE ~0U
#define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b)))
#define VF_POINT_GET_PSTATE(a) (((a)->pstates) ?\
__fls((a)->pstates) :\
VF_POINT_INVALID_PSTATE)
#define VF_POINT_COMMON_PSTATE(a, b) (((a)->pstates & (b)->pstates) ?\
__fls((a)->pstates & (b)->pstates) :\
VF_POINT_INVALID_PSTATE)
/*
* These events, defined in common code are the counterparts of the uapi
* events. There should be a conversion function to take care to convert
* these to the uapi events.
*/
/* Event associated to a VF update */
#define NVGPU_EVENT_VF_UPDATE 0
/* Recoverable alarms (POLLPRI) */
/* Alarm when target frequency on any session is not possible */
#define NVGPU_EVENT_ALARM_TARGET_VF_NOT_POSSIBLE 1
/* Alarm when target frequency on current session is not possible */
#define NVGPU_EVENT_ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE 2
/* Alarm when Clock Arbiter failed */
#define NVGPU_EVENT_ALARM_CLOCK_ARBITER_FAILED 3
/* Alarm when VF table update failed */
#define NVGPU_EVENT_ALARM_VF_TABLE_UPDATE_FAILED 4
/* Alarm on thermal condition */
#define NVGPU_EVENT_ALARM_THERMAL_ABOVE_THRESHOLD 5
/* Alarm on power condition */
#define NVGPU_EVENT_ALARM_POWER_ABOVE_THRESHOLD 6
/* Non recoverable alarm (POLLHUP) */
/* Alarm on GPU shutdown/fall from bus */
#define NVGPU_EVENT_ALARM_GPU_LOST 7
#define NVGPU_EVENT_LAST NVGPU_EVENT_ALARM_GPU_LOST
/* Local Alarms */
#define EVENT(alarm) (0x1UL << NVGPU_EVENT_##alarm)
#define LOCAL_ALARM_MASK (EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE) | \
EVENT(VF_UPDATE))
#define _WRAPGTEQ(a, b) ((a-b) > 0)
/*
* NVGPU_POLL* defines equivalent to the POLL* linux defines
*/
#define NVGPU_POLLIN (1 << 0)
#define NVGPU_POLLPRI (1 << 1)
#define NVGPU_POLLOUT (1 << 2)
#define NVGPU_POLLRDNORM (1 << 3)
#define NVGPU_POLLHUP (1 << 4)
/* NVGPU_CLK_DOMAIN_* defines equivalent to NVGPU_GPU_CLK_DOMAIN_*
* defines in uapi header
*/
/* Memory clock */
#define NVGPU_CLK_DOMAIN_MCLK (0)
/* Main graphics core clock */
#define NVGPU_CLK_DOMAIN_GPCCLK (1)
#define NVGPU_CLK_DOMAIN_MAX (NVGPU_CLK_DOMAIN_GPCCLK)
#define clk_arb_dbg(g, fmt, args...) \
do { \
nvgpu_log(g, gpu_dbg_clk_arb, \
fmt, ##args); \
} while (0)
struct nvgpu_clk_notification {
u32 notification;
u64 timestamp;
};
struct nvgpu_clk_notification_queue {
u32 size;
nvgpu_atomic_t head;
nvgpu_atomic_t tail;
struct nvgpu_clk_notification *notifications;
};
struct nvgpu_clk_vf_point {
u16 pstates;
union {
struct {
u16 gpc_mhz;
u16 sys_mhz;
u16 xbar_mhz;
};
u16 mem_mhz;
};
u32 uvolt;
u32 uvolt_sram;
};
struct nvgpu_clk_vf_table {
u32 mclk_num_points;
struct nvgpu_clk_vf_point *mclk_points;
u32 gpc2clk_num_points;
struct nvgpu_clk_vf_point *gpc2clk_points;
};
#ifdef CONFIG_DEBUG_FS
struct nvgpu_clk_arb_debug {
s64 switch_max;
s64 switch_min;
u64 switch_num;
s64 switch_avg;
s64 switch_std;
};
#endif
struct nvgpu_clk_arb_target {
u16 mclk;
u16 gpc2clk;
u32 pstate;
};
enum clk_arb_work_item_type {
CLK_ARB_WORK_UPDATE_VF_TABLE,
CLK_ARB_WORK_UPDATE_ARB
};
struct nvgpu_clk_arb_work_item {
enum clk_arb_work_item_type item_type;
struct nvgpu_clk_arb *arb;
struct nvgpu_list_node worker_item;
};
struct nvgpu_clk_arb {
struct nvgpu_spinlock sessions_lock;
struct nvgpu_spinlock users_lock;
struct nvgpu_spinlock requests_lock;
struct nvgpu_mutex pstate_lock;
struct nvgpu_list_node users;
struct nvgpu_list_node sessions;
struct nvgpu_list_node requests;
struct gk20a *g;
int status;
struct nvgpu_clk_arb_target actual_pool[2];
struct nvgpu_clk_arb_target *actual;
u16 gpc2clk_default_mhz;
u16 mclk_default_mhz;
u32 voltuv_actual;
u16 gpc2clk_min, gpc2clk_max;
u16 mclk_min, mclk_max;
struct nvgpu_clk_arb_work_item update_vf_table_work_item;
struct nvgpu_clk_arb_work_item update_arb_work_item;
struct nvgpu_cond request_wq;
struct nvgpu_clk_vf_table *current_vf_table;
struct nvgpu_clk_vf_table vf_table_pool[2];
u32 vf_table_index;
u16 *mclk_f_points;
nvgpu_atomic_t req_nr;
u32 mclk_f_numpoints;
u16 *gpc2clk_f_points;
u32 gpc2clk_f_numpoints;
nvgpu_atomic64_t alarm_mask;
struct nvgpu_clk_notification_queue notification_queue;
#ifdef CONFIG_DEBUG_FS
struct nvgpu_clk_arb_debug debug_pool[2];
struct nvgpu_clk_arb_debug *debug;
bool debugfs_set;
#endif
};
struct nvgpu_clk_dev {
struct nvgpu_clk_session *session;
union {
struct nvgpu_list_node link;
struct nvgpu_list_node node;
};
struct nvgpu_cond readout_wq;
nvgpu_atomic_t poll_mask;
u16 gpc2clk_target_mhz;
u16 mclk_target_mhz;
u32 alarms_reported;
nvgpu_atomic_t enabled_mask;
struct nvgpu_clk_notification_queue queue;
u32 arb_queue_head;
struct nvgpu_ref refcount;
};
struct nvgpu_clk_session {
bool zombie;
struct gk20a *g;
struct nvgpu_ref refcount;
struct nvgpu_list_node link;
struct nvgpu_list_node targets;
struct nvgpu_spinlock session_lock;
struct nvgpu_clk_arb_target target_pool[2];
struct nvgpu_clk_arb_target *target;
};
static inline struct nvgpu_clk_session *
nvgpu_clk_session_from_link(struct nvgpu_list_node *node)
{
return (struct nvgpu_clk_session *)
((uintptr_t)node - offsetof(struct nvgpu_clk_session, link));
};
static inline struct nvgpu_clk_dev *
nvgpu_clk_dev_from_node(struct nvgpu_list_node *node)
{
return (struct nvgpu_clk_dev *)
((uintptr_t)node - offsetof(struct nvgpu_clk_dev, node));
};
static inline struct nvgpu_clk_dev *
nvgpu_clk_dev_from_link(struct nvgpu_list_node *node)
{
return (struct nvgpu_clk_dev *)
((uintptr_t)node - offsetof(struct nvgpu_clk_dev, link));
};
static inline struct nvgpu_clk_arb_work_item *
nvgpu_clk_arb_work_item_from_worker_item(struct nvgpu_list_node *node)
{
return (struct nvgpu_clk_arb_work_item *)
((uintptr_t)node - offsetof(struct nvgpu_clk_arb_work_item, worker_item));
};
void nvgpu_clk_arb_worker_enqueue(struct gk20a *g,
struct nvgpu_clk_arb_work_item *work_item);
int nvgpu_clk_arb_init_arbiter(struct gk20a *g);
int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
u16 *min_mhz, u16 *max_mhz);
int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g,
u32 api_domain, u16 *actual_mhz);
int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g,
u32 api_domain, u16 *effective_mhz);
int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g,
u32 api_domain, u32 *max_points, u16 *fpoints);
u32 nvgpu_clk_arb_get_arbiter_clk_domains(struct gk20a *g);
bool nvgpu_clk_arb_is_valid_domain(struct gk20a *g, u32 api_domain);
void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g);
int nvgpu_clk_arb_install_session_fd(struct gk20a *g,
struct nvgpu_clk_session *session);
int nvgpu_clk_arb_init_session(struct gk20a *g,
struct nvgpu_clk_session **_session);
void nvgpu_clk_arb_release_session(struct gk20a *g,
struct nvgpu_clk_session *session);
int nvgpu_clk_arb_commit_request_fd(struct gk20a *g,
struct nvgpu_clk_session *session, int request_fd);
int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session,
int fd, u32 api_domain, u16 target_mhz);
int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session,
u32 api_domain, u16 *target_mhz);
int nvgpu_clk_arb_install_event_fd(struct gk20a *g,
struct nvgpu_clk_session *session, int *event_fd, u32 alarm_mask);
int nvgpu_clk_arb_install_request_fd(struct gk20a *g,
struct nvgpu_clk_session *session, int *event_fd);
void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g);
int nvgpu_clk_arb_get_current_pstate(struct gk20a *g);
void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock);
void nvgpu_clk_arb_send_thermal_alarm(struct gk20a *g);
void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm);
void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount);
void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount);
int nvgpu_clk_notification_queue_alloc(struct gk20a *g,
struct nvgpu_clk_notification_queue *queue,
size_t events_number);
void nvgpu_clk_notification_queue_free(struct gk20a *g,
struct nvgpu_clk_notification_queue *queue);
void nvgpu_clk_arb_event_post_event(struct nvgpu_clk_dev *dev);
#ifdef CONFIG_DEBUG_FS
int nvgpu_clk_arb_debugfs_init(struct gk20a *g);
#endif
#endif /* __NVGPU_CLK_ARB_H__ */
|