/*
* Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef NVGPU_CLK_ARB_H
#define NVGPU_CLK_ARB_H
struct gk20a;
#include <nvgpu/types.h>
#include <nvgpu/bitops.h>
#include <nvgpu/lock.h>
#include <nvgpu/kmem.h>
#include <nvgpu/atomic.h>
#include <nvgpu/bug.h>
#include <nvgpu/kref.h>
#include <nvgpu/log.h>
#include <nvgpu/barrier.h>
#include <nvgpu/cond.h>
#include "clk/clk.h"
#include "pstate/pstate.h"
#include "lpwr/lpwr.h"
#include "volt/volt.h"
#define MAX_F_POINTS 256
#define DEFAULT_EVENT_NUMBER 32
struct nvgpu_clk_dev;
struct nvgpu_clk_arb_target;
struct nvgpu_clk_notification_queue;
struct nvgpu_clk_session;
#define VF_POINT_INVALID_PSTATE ~0U
#define VF_POINT_SET_PSTATE_SUPPORTED(a, b) ((a)->pstates |= (1UL << (b)))
#define VF_POINT_GET_PSTATE(a) (((a)->pstates) ?\
__fls((a)->pstates) :\
VF_POINT_INVALID_PSTATE)
#define VF_POINT_COMMON_PSTATE(a, b) (((a)->pstates & (b)->pstates) ?\
__fls((a)->pstates & (b)->pstates) :\
VF_POINT_INVALID_PSTATE)
/*
* These events, defined in common code are the counterparts of the uapi
* events. There should be a conversion function to take care to convert
* these to the uapi events.
*/
/* Event associated to a VF update */
#define NVGPU_EVENT_VF_UPDATE 0
/* Recoverable alarms (POLLPRI) */
/* Alarm when target frequency on any session is not possible */
#define NVGPU_EVENT_ALARM_TARGET_VF_NOT_POSSIBLE 1
/* Alarm when target frequency on current session is not possible */
#define NVGPU_EVENT_ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE 2
/* Alarm when Clock Arbiter failed */
#define NVGPU_EVENT_ALARM_CLOCK_ARBITER_FAILED 3
/* Alarm when VF table update failed */
#define NVGPU_EVENT_ALARM_VF_TABLE_UPDATE_FAILED 4
/* Alarm on thermal condition */
#define NVGPU_EVENT_ALARM_THERMAL_ABOVE_THRESHOLD 5
/* Alarm on power condition */
#define NVGPU_EVENT_ALARM_POWER_ABOVE_THRESHOLD 6
/* Non recoverable alarm (POLLHUP) */
/* Alarm on GPU shutdown/fall from bus */
#define NVGPU_EVENT_ALARM_GPU_LOST 7
#define NVGPU_EVENT_LAST NVGPU_EVENT_ALARM_GPU_LOST
/* Local Alarms */
#define EVENT(alarm) (0x1UL << NVGPU_EVENT_##alarm)
#define LOCAL_ALARM_MASK (EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE) | \
EVENT(VF_UPDATE))
#define _WRAPGTEQ(a, b) ((a-b) > 0)
/*
* NVGPU_POLL* defines equivalent to the POLL* linux defines
*/
#define NVGPU_POLLIN (1 << 0)
#define NVGPU_POLLPRI (1 << 1)
#define NVGPU_POLLOUT (1 << 2)
#define NVGPU_POLLRDNORM (1 << 3)
#define NVGPU_POLLHUP (1 << 4)
/* NVGPU_CLK_DOMAIN_* defines equivalent to NVGPU_GPU_CLK_DOMAIN_*
* defines in uapi header
*/
/* Memory clock */
#define NVGPU_CLK_DOMAIN_MCLK (0)
/* Main graphics core clock */
#define NVGPU_CLK_DOMAIN_GPCCLK (1)
#define NVGPU_CLK_DOMAIN_MAX (NVGPU_CLK_DOMAIN_GPCCLK)
#define clk_arb_dbg(g, fmt, args...) \
do { \
nvgpu_log(g, gpu_dbg_clk_arb, \
fmt, ##args); \
} while (0)
struct nvgpu_clk_notification {
u32 notification;
u64 timestamp;
};
struct nvgpu_clk_notification_queue {
u32 size;
nvgpu_atomic_t head;
nvgpu_atomic_t tail;
struct nvgpu_clk_notification *notifications;
};
struct nvgpu_clk_vf_point {
u16 pstates;
union {
struct {
u16 gpc_mhz;
u16 sys_mhz;
u16 xbar_mhz;
};
u16 mem_mhz;
};
u32 uvolt;
u32 uvolt_sram;
};
struct nvgpu_clk_vf_table {
u32 mclk_num_points;
struct nvgpu_clk_vf_point *mclk_points;
u32 gpc2clk_num_points;
struct nvgpu_clk_vf_point *gpc2clk_points;
};
#ifdef CONFIG_DEBUG_FS
struct nvgpu_clk_arb_debug {
s64 switch_max;
s64 switch_min;
u64 switch_num;
s64 switch_avg;
s64 switch_std;
};
#endif
struct nvgpu_clk_arb_target {
u16 mclk;
u16 gpc2clk;
u32 pstate;
};
enum clk_arb_work_item_type {
CLK_ARB_WORK_UPDATE_VF_TABLE,
CLK_ARB_WORK_UPDATE_ARB
};
struct nvgpu_clk_arb_work_item {
enum clk_arb_work_item_type item_type;
struct nvgpu_clk_arb *arb;
struct nvgpu_list_node worker_item;
};
struct nvgpu_clk_arb {
struct nvgpu_spinlock sessions_lock;
struct nvgpu_spinlock users_lock;
struct nvgpu_spinlock requests_lock;
struct nvgpu_mutex pstate_lock;
struct nvgpu_list_node users;
struct nvgpu_list_node sessions;
struct nvgpu_list_node requests;
struct gk20a *g;
int status;
struct nvgpu_clk_arb_target actual_pool[2];
struct nvgpu_clk_arb_target *actual;
u16 gpc2clk_default_mhz;
u16 mclk_default_mhz;
u32 voltuv_actual;
u16 gpc2clk_min, gpc2clk_max;
u16 mclk_min, mclk_max;
struct nvgpu_clk_arb_work_item update_vf_table_work_item;
struct nvgpu_clk_arb_work_item update_arb_work_item;
struct nvgpu_cond request_wq;
struct nvgpu_clk_vf_table *current_vf_table;
struct nvgpu_clk_vf_table vf_table_pool[2];
u32 vf_table_index;
u16 *mclk_f_points;
nvgpu_atomic_t req_nr;
u32 mclk_f_numpoints;
u16 *gpc2clk_f_points;
u32 gpc2clk_f_numpoints;
bool clk_arb_events_supported;
nvgpu_atomic64_t alarm_mask;
struct nvgpu_clk_notification_queue notification_queue;
#ifdef CONFIG_DEBUG_FS
struct nvgpu_clk_arb_debug debug_pool[2];
struct nvgpu_clk_arb_debug *debug;
bool debugfs_set;
#endif
};
struct nvgpu_clk_dev {
struct nvgpu_clk_session *session;
union {
struct nvgpu_list_node link;
struct nvgpu_list_node node;
};
struct nvgpu_cond readout_wq;
nvgpu_atomic_t poll_mask;
u16 gpc2clk_target_mhz;
u16 mclk_target_mhz;
u32 alarms_reported;
nvgpu_atomic_t enabled_mask;
struct nvgpu_clk_notification_queue queue;
u32 arb_queue_head;
struct nvgpu_ref refcount;
};
struct nvgpu_clk_session {
bool zombie;
struct gk20a *g;
struct nvgpu_ref refcount;
struct nvgpu_list_node link;
struct nvgpu_list_node targets;
struct nvgpu_spinlock session_lock;
struct nvgpu_clk_arb_target target_pool[2];
struct nvgpu_clk_arb_target *target;
};
static inline struct nvgpu_clk_session *
nvgpu_clk_session_from_link(struct nvgpu_list_node *node)
{
return (struct nvgpu_clk_session *)
((uintptr_t)node - offsetof(struct nvgpu_clk_session, link));
};
static inline struct nvgpu_clk_dev *
nvgpu_clk_dev_from_node(struct nvgpu_list_node *node)
{
return (struct nvgpu_clk_dev *)
((uintptr_t)node - offsetof(struct nvgpu_clk_dev, node));
};
static inline struct nvgpu_clk_dev *
nvgpu_clk_dev_from_link(struct nvgpu_list_node *node)
{
return (struct nvgpu_clk_dev *)
((uintptr_t)node - offsetof(struct nvgpu_clk_dev, link));
};
static inline struct nvgpu_clk_arb_work_item *
nvgpu_clk_arb_work_item_from_worker_item(struct nvgpu_list_node *node)
{
return (struct nvgpu_clk_arb_work_item *)
((uintptr_t)node - offsetof(struct nvgpu_clk_arb_work_item, worker_item));
};
void nvgpu_clk_arb_worker_enqueue(struct gk20a *g,
struct nvgpu_clk_arb_work_item *work_item);
int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb);
int nvgpu_clk_arb_worker_init(struct gk20a *g);
int nvgpu_clk_arb_init_arbiter(struct gk20a *g);
bool nvgpu_clk_arb_has_active_req(struct gk20a *g);
int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
u16 *min_mhz, u16 *max_mhz);
int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g,
u32 api_domain, u16 *actual_mhz);
int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g,
u32 api_domain, u16 *effective_mhz);
int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g,
u32 api_domain, u32 *max_points, u16 *fpoints);
u32 nvgpu_clk_arb_get_arbiter_clk_domains(struct gk20a *g);
bool nvgpu_clk_arb_is_valid_domain(struct gk20a *g, u32 api_domain);
void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g);
int nvgpu_clk_arb_install_session_fd(struct gk20a *g,
struct nvgpu_clk_session *session);
int nvgpu_clk_arb_init_session(struct gk20a *g,
struct nvgpu_clk_session **_session);
void nvgpu_clk_arb_release_session(struct gk20a *g,
struct nvgpu_clk_session *session);
int nvgpu_clk_arb_commit_request_fd(struct gk20a *g,
struct nvgpu_clk_session *session, int request_fd);
int nvgpu_clk_arb_set_session_target_mhz(struct nvgpu_clk_session *session,
int fd, u32 api_domain, u16 target_mhz);
int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session,
u32 api_domain, u16 *target_mhz);
int nvgpu_clk_arb_install_event_fd(struct gk20a *g,
struct nvgpu_clk_session *session, int *event_fd, u32 alarm_mask);
int nvgpu_clk_arb_install_request_fd(struct gk20a *g,
struct nvgpu_clk_session *session, int *event_fd);
void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g);
int nvgpu_clk_arb_get_current_pstate(struct gk20a *g);
void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock);
void nvgpu_clk_arb_send_thermal_alarm(struct gk20a *g);
void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm);
void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm);
void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm);
void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount);
void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount);
u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev,
struct nvgpu_clk_arb_target *target,
u32 alarm);
int nvgpu_clk_notification_queue_alloc(struct gk20a *g,
struct nvgpu_clk_notification_queue *queue,
size_t events_number);
void nvgpu_clk_notification_queue_free(struct gk20a *g,
struct nvgpu_clk_notification_queue *queue);
void nvgpu_clk_arb_event_post_event(struct nvgpu_clk_dev *dev);
unsigned long nvgpu_clk_measure_freq(struct gk20a *g, u32 api_domain);
#ifdef CONFIG_DEBUG_FS
int nvgpu_clk_arb_debugfs_init(struct gk20a *g);
#endif
#endif /* NVGPU_CLK_ARB_H */
