From 01e6fac4d61fdd7fff5433942ec93fc2ea1e4df1 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Wed, 28 Jun 2023 18:24:25 -0400 Subject: Include nvgpu headers These are needed to build on NVIDIA's Jetson boards for the time being. Only a couple structs are required, so it should be fairly easy to remove this dependency at some point in the future. --- include/clk/clk.c | 942 +++++++++++++++++++++ include/clk/clk.h | 144 ++++ include/clk/clk_arb.c | 1087 ++++++++++++++++++++++++ include/clk/clk_domain.c | 1666 +++++++++++++++++++++++++++++++++++++ include/clk/clk_domain.h | 157 ++++ include/clk/clk_fll.c | 495 +++++++++++ include/clk/clk_fll.h | 81 ++ include/clk/clk_freq_controller.c | 462 ++++++++++ include/clk/clk_freq_controller.h | 84 ++ include/clk/clk_mclk.h | 60 ++ include/clk/clk_prog.c | 1152 +++++++++++++++++++++++++ include/clk/clk_prog.h | 100 +++ include/clk/clk_vf_point.c | 433 ++++++++++ include/clk/clk_vf_point.h | 83 ++ include/clk/clk_vin.c | 573 +++++++++++++ include/clk/clk_vin.h | 79 ++ 16 files changed, 7598 insertions(+) create mode 100644 include/clk/clk.c create mode 100644 include/clk/clk.h create mode 100644 include/clk/clk_arb.c create mode 100644 include/clk/clk_domain.c create mode 100644 include/clk/clk_domain.h create mode 100644 include/clk/clk_fll.c create mode 100644 include/clk/clk_fll.h create mode 100644 include/clk/clk_freq_controller.c create mode 100644 include/clk/clk_freq_controller.h create mode 100644 include/clk/clk_mclk.h create mode 100644 include/clk/clk_prog.c create mode 100644 include/clk/clk_prog.h create mode 100644 include/clk/clk_vf_point.c create mode 100644 include/clk/clk_vf_point.h create mode 100644 include/clk/clk_vin.c create mode 100644 include/clk/clk_vin.h (limited to 'include/clk') diff --git a/include/clk/clk.c b/include/clk/clk.c new file mode 100644 index 0000000..d8e30c4 --- /dev/null +++ b/include/clk/clk.c @@ -0,0 +1,942 @@ +/* + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include + +#include "clk.h" +#include "ctrl/ctrlclk.h" +#include "ctrl/ctrlvolt.h" +#include "volt/volt.h" + +#define BOOT_GPC2CLK_MHZ 2581 +#define BOOT_MCLK_MHZ 3003 + +struct clkrpc_pmucmdhandler_params { + struct nv_pmu_clk_rpc *prpccall; + u32 success; +}; + +static void clkrpc_pmucmdhandler(struct gk20a *g, struct pmu_msg *msg, + void *param, u32 handle, u32 status) +{ + struct clkrpc_pmucmdhandler_params *phandlerparams = + (struct clkrpc_pmucmdhandler_params *)param; + + nvgpu_log_info(g, " "); + + if (msg->msg.clk.msg_type != NV_PMU_CLK_MSG_ID_RPC) { + nvgpu_err(g, "unsupported msg for VFE LOAD RPC %x", + msg->msg.clk.msg_type); + return; + } + + if (phandlerparams->prpccall->b_supported) { + phandlerparams->success = 1; + } +} + + +int clk_pmu_freq_effective_avg_load(struct gk20a *g, bool bload) +{ + struct pmu_cmd cmd; + struct pmu_payload payload; + u32 status; + u32 seqdesc; + struct nv_pmu_clk_rpc rpccall; + struct clkrpc_pmucmdhandler_params handler; + struct nv_pmu_clk_load *clkload; + + memset(&payload, 0, sizeof(struct pmu_payload)); + memset(&rpccall, 0, sizeof(struct nv_pmu_clk_rpc)); + memset(&handler, 0, sizeof(struct clkrpc_pmucmdhandler_params)); + memset(&cmd, 0, sizeof(struct pmu_cmd)); + + rpccall.function = NV_PMU_CLK_RPC_ID_LOAD; + clkload = &rpccall.params.clk_load; + clkload->feature = NV_NV_PMU_CLK_LOAD_FEATURE_FREQ_EFFECTIVE_AVG; + clkload->action_mask = bload ? + NV_NV_PMU_CLK_LOAD_ACTION_MASK_FREQ_EFFECTIVE_AVG_CALLBACK_YES : + NV_NV_PMU_CLK_LOAD_ACTION_MASK_FREQ_EFFECTIVE_AVG_CALLBACK_NO; + + cmd.hdr.unit_id = PMU_UNIT_CLK; + cmd.hdr.size = (u32)sizeof(struct nv_pmu_clk_cmd) + + (u32)sizeof(struct pmu_hdr); + + cmd.cmd.clk.cmd_type = NV_PMU_CLK_CMD_ID_RPC; + + payload.in.buf = (u8 *)&rpccall; + payload.in.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.in.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.in.offset = NV_PMU_CLK_CMD_RPC_ALLOC_OFFSET; + + payload.out.buf = (u8 *)&rpccall; + payload.out.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.out.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.out.offset = NV_PMU_CLK_MSG_RPC_ALLOC_OFFSET; + + handler.prpccall = &rpccall; + handler.success = 0; + + status = nvgpu_pmu_cmd_post(g, &cmd, NULL, &payload, + PMU_COMMAND_QUEUE_LPQ, + clkrpc_pmucmdhandler, (void *)&handler, + &seqdesc, ~0); + if (status) { + nvgpu_err(g, "unable to post clk RPC cmd %x", + cmd.cmd.clk.cmd_type); + goto done; + } + + pmu_wait_message_cond(&g->pmu, + gk20a_get_gr_idle_timeout(g), + &handler.success, 1); + if (handler.success == 0) { + nvgpu_err(g, "rpc call to load Effective avg clk domain freq failed"); + status = -EINVAL; + } + +done: + return status; +} + +u32 clk_freq_effective_avg(struct gk20a *g, u32 clkDomainMask) { + + struct pmu_cmd cmd; + struct pmu_payload payload; + u32 status; + u32 seqdesc; + struct nv_pmu_clk_rpc rpccall; + struct clkrpc_pmucmdhandler_params handler; + struct nv_pmu_clk_freq_effective_avg *clk_freq_effective_avg; + + memset(&payload, 0, sizeof(struct pmu_payload)); + memset(&rpccall, 0, sizeof(struct nv_pmu_clk_rpc)); + memset(&handler, 0, sizeof(struct clkrpc_pmucmdhandler_params)); + memset(&cmd, 0, sizeof(struct pmu_cmd)); + + rpccall.function = NV_PMU_CLK_RPC_ID_CLK_FREQ_EFF_AVG; + clk_freq_effective_avg = &rpccall.params.clk_freq_effective_avg; + clk_freq_effective_avg->clkDomainMask = clkDomainMask; + + cmd.hdr.unit_id = PMU_UNIT_CLK; + cmd.hdr.size = (u32)sizeof(struct nv_pmu_clk_cmd) + + (u32)sizeof(struct pmu_hdr); + + cmd.cmd.clk.cmd_type = NV_PMU_CLK_CMD_ID_RPC; + + payload.in.buf = (u8 *)&rpccall; + payload.in.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.in.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.in.offset = NV_PMU_CLK_CMD_RPC_ALLOC_OFFSET; + + payload.out.buf = (u8 *)&rpccall; + payload.out.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.out.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.out.offset = NV_PMU_CLK_MSG_RPC_ALLOC_OFFSET; + + handler.prpccall = &rpccall; + handler.success = 0; + + status = nvgpu_pmu_cmd_post(g, &cmd, NULL, &payload, + PMU_COMMAND_QUEUE_LPQ, + clkrpc_pmucmdhandler, (void *)&handler, + &seqdesc, ~0); + if (status) { + nvgpu_err(g, "unable to post clk RPC cmd %x", + cmd.cmd.clk.cmd_type); + goto done; + } + + pmu_wait_message_cond(&g->pmu, + gk20a_get_gr_idle_timeout(g), + &handler.success, 1); + if (handler.success == 0) { + nvgpu_err(g, "rpc call to get clk frequency average failed"); + status = -EINVAL; + goto done; + } + + return rpccall.params.clk_freq_effective_avg.freqkHz[clkDomainMask]; + +done: + return status; +} + +int clk_pmu_freq_controller_load(struct gk20a *g, bool bload, u8 bit_idx) +{ + struct pmu_cmd cmd; + struct pmu_payload payload; + u32 status; + u32 seqdesc; + struct nv_pmu_clk_rpc rpccall; + struct clkrpc_pmucmdhandler_params handler; + struct nv_pmu_clk_load *clkload; + struct clk_freq_controllers *pclk_freq_controllers; + struct ctrl_boardobjgrp_mask_e32 *load_mask; + struct boardobjgrpmask_e32 isolate_cfc_mask; + + memset(&payload, 0, sizeof(struct pmu_payload)); + memset(&rpccall, 0, sizeof(struct nv_pmu_clk_rpc)); + memset(&handler, 0, sizeof(struct clkrpc_pmucmdhandler_params)); + + pclk_freq_controllers = &g->clk_pmu.clk_freq_controllers; + rpccall.function = NV_PMU_CLK_RPC_ID_LOAD; + clkload = &rpccall.params.clk_load; + clkload->feature = NV_NV_PMU_CLK_LOAD_FEATURE_FREQ_CONTROLLER; + clkload->action_mask = bload ? + NV_NV_PMU_CLK_LOAD_ACTION_MASK_FREQ_CONTROLLER_CALLBACK_YES : + NV_NV_PMU_CLK_LOAD_ACTION_MASK_FREQ_CONTROLLER_CALLBACK_NO; + + load_mask = &rpccall.params.clk_load.payload.freq_controllers.load_mask; + + status = boardobjgrpmask_e32_init(&isolate_cfc_mask, NULL); + + if (bit_idx == CTRL_CLK_CLK_FREQ_CONTROLLER_ID_ALL) { + status = boardobjgrpmask_export( + &pclk_freq_controllers-> + freq_ctrl_load_mask.super, + pclk_freq_controllers-> + freq_ctrl_load_mask.super.bitcount, + &load_mask->super); + + + } else { + status = boardobjgrpmask_bitset(&isolate_cfc_mask.super, + bit_idx); + status = boardobjgrpmask_export(&isolate_cfc_mask.super, + isolate_cfc_mask.super.bitcount, + &load_mask->super); + if (bload) { + status = boardobjgrpmask_bitset( + &pclk_freq_controllers-> + freq_ctrl_load_mask.super, + bit_idx); + } else { + status = boardobjgrpmask_bitclr( + &pclk_freq_controllers-> + freq_ctrl_load_mask.super, + bit_idx); + } + } + + if (status) { + nvgpu_err(g, "Error in generating mask used to select CFC"); + goto done; + } + + cmd.hdr.unit_id = PMU_UNIT_CLK; + cmd.hdr.size = (u32)sizeof(struct nv_pmu_clk_cmd) + + (u32)sizeof(struct pmu_hdr); + + cmd.cmd.clk.cmd_type = NV_PMU_CLK_CMD_ID_RPC; + + payload.in.buf = (u8 *)&rpccall; + payload.in.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.in.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.in.offset = NV_PMU_CLK_CMD_RPC_ALLOC_OFFSET; + + payload.out.buf = (u8 *)&rpccall; + payload.out.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.out.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.out.offset = NV_PMU_CLK_MSG_RPC_ALLOC_OFFSET; + + handler.prpccall = &rpccall; + handler.success = 0; + status = nvgpu_pmu_cmd_post(g, &cmd, NULL, &payload, + PMU_COMMAND_QUEUE_LPQ, + clkrpc_pmucmdhandler, (void *)&handler, + &seqdesc, ~0); + + if (status) { + nvgpu_err(g, "unable to post clk RPC cmd %x", + cmd.cmd.clk.cmd_type); + goto done; + } + + pmu_wait_message_cond(&g->pmu, + gk20a_get_gr_idle_timeout(g), + &handler.success, 1); + + if (handler.success == 0) { + nvgpu_err(g, "rpc call to load freq cntlr cal failed"); + status = -EINVAL; + } + +done: + return status; +} + +u32 clk_pmu_vin_load(struct gk20a *g) +{ + struct pmu_cmd cmd; + struct pmu_payload payload; + u32 status; + u32 seqdesc; + struct nv_pmu_clk_rpc rpccall; + struct clkrpc_pmucmdhandler_params handler; + struct nv_pmu_clk_load *clkload; + + memset(&payload, 0, sizeof(struct pmu_payload)); + memset(&rpccall, 0, sizeof(struct nv_pmu_clk_rpc)); + memset(&handler, 0, sizeof(struct clkrpc_pmucmdhandler_params)); + + rpccall.function = NV_PMU_CLK_RPC_ID_LOAD; + clkload = &rpccall.params.clk_load; + clkload->feature = NV_NV_PMU_CLK_LOAD_FEATURE_VIN; + clkload->action_mask = NV_NV_PMU_CLK_LOAD_ACTION_MASK_VIN_HW_CAL_PROGRAM_YES << 4; + + cmd.hdr.unit_id = PMU_UNIT_CLK; + cmd.hdr.size = (u32)sizeof(struct nv_pmu_clk_cmd) + + (u32)sizeof(struct pmu_hdr); + + cmd.cmd.clk.cmd_type = NV_PMU_CLK_CMD_ID_RPC; + cmd.cmd.clk.generic.b_perf_daemon_cmd =false; + + payload.in.buf = (u8 *)&rpccall; + payload.in.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.in.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.in.offset = NV_PMU_CLK_CMD_RPC_ALLOC_OFFSET; + + payload.out.buf = (u8 *)&rpccall; + payload.out.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.out.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.out.offset = NV_PMU_CLK_MSG_RPC_ALLOC_OFFSET; + + handler.prpccall = &rpccall; + handler.success = 0; + status = nvgpu_pmu_cmd_post(g, &cmd, NULL, &payload, + PMU_COMMAND_QUEUE_LPQ, + clkrpc_pmucmdhandler, (void *)&handler, + &seqdesc, ~0); + + if (status) { + nvgpu_err(g, "unable to post clk RPC cmd %x", + cmd.cmd.clk.cmd_type); + goto done; + } + + pmu_wait_message_cond(&g->pmu, + gk20a_get_gr_idle_timeout(g), + &handler.success, 1); + + if (handler.success == 0) { + nvgpu_err(g, "rpc call to load vin cal failed"); + status = -EINVAL; + } + +done: + return status; +} + +u32 nvgpu_clk_vf_change_inject_data_fill_gp10x(struct gk20a *g, + struct nv_pmu_clk_rpc *rpccall, + struct set_fll_clk *setfllclk) +{ + struct nv_pmu_clk_vf_change_inject *vfchange; + + vfchange = &rpccall->params.clk_vf_change_inject; + vfchange->flags = 0; + vfchange->clk_list.num_domains = 3; + vfchange->clk_list.clk_domains[0].clk_domain = CTRL_CLK_DOMAIN_GPC2CLK; + vfchange->clk_list.clk_domains[0].clk_freq_khz = + setfllclk->gpc2clkmhz * 1000; + vfchange->clk_list.clk_domains[0].clk_flags = 0; + vfchange->clk_list.clk_domains[0].current_regime_id = + setfllclk->current_regime_id_gpc; + vfchange->clk_list.clk_domains[0].target_regime_id = + setfllclk->target_regime_id_gpc; + vfchange->clk_list.clk_domains[1].clk_domain = CTRL_CLK_DOMAIN_XBAR2CLK; + vfchange->clk_list.clk_domains[1].clk_freq_khz = + setfllclk->xbar2clkmhz * 1000; + vfchange->clk_list.clk_domains[1].clk_flags = 0; + vfchange->clk_list.clk_domains[1].current_regime_id = + setfllclk->current_regime_id_xbar; + vfchange->clk_list.clk_domains[1].target_regime_id = + setfllclk->target_regime_id_xbar; + vfchange->clk_list.clk_domains[2].clk_domain = CTRL_CLK_DOMAIN_SYS2CLK; + vfchange->clk_list.clk_domains[2].clk_freq_khz = + setfllclk->sys2clkmhz * 1000; + vfchange->clk_list.clk_domains[2].clk_flags = 0; + vfchange->clk_list.clk_domains[2].current_regime_id = + setfllclk->current_regime_id_sys; + vfchange->clk_list.clk_domains[2].target_regime_id = + setfllclk->target_regime_id_sys; + vfchange->volt_list.num_rails = 1; + vfchange->volt_list.rails[0].volt_domain = CTRL_VOLT_DOMAIN_LOGIC; + vfchange->volt_list.rails[0].voltage_uv = setfllclk->voltuv; + vfchange->volt_list.rails[0].voltage_min_noise_unaware_uv = + setfllclk->voltuv; + + return 0; +} + +u32 nvgpu_clk_vf_change_inject_data_fill_gv10x(struct gk20a *g, + struct nv_pmu_clk_rpc *rpccall, + struct set_fll_clk *setfllclk) +{ + struct nv_pmu_clk_vf_change_inject_v1 *vfchange; + + vfchange = &rpccall->params.clk_vf_change_inject_v1; + vfchange->flags = 0; + vfchange->clk_list.num_domains = 4; + vfchange->clk_list.clk_domains[0].clk_domain = CTRL_CLK_DOMAIN_GPCCLK; + vfchange->clk_list.clk_domains[0].clk_freq_khz = + setfllclk->gpc2clkmhz * 1000; + + vfchange->clk_list.clk_domains[1].clk_domain = CTRL_CLK_DOMAIN_XBARCLK; + vfchange->clk_list.clk_domains[1].clk_freq_khz = + setfllclk->xbar2clkmhz * 1000; + + vfchange->clk_list.clk_domains[2].clk_domain = CTRL_CLK_DOMAIN_SYSCLK; + vfchange->clk_list.clk_domains[2].clk_freq_khz = + setfllclk->sys2clkmhz * 1000; + + vfchange->clk_list.clk_domains[3].clk_domain = CTRL_CLK_DOMAIN_NVDCLK; + vfchange->clk_list.clk_domains[3].clk_freq_khz = 855 * 1000; + + vfchange->volt_list.num_rails = 1; + vfchange->volt_list.rails[0].rail_idx = 0; + vfchange->volt_list.rails[0].voltage_uv = setfllclk->voltuv; + vfchange->volt_list.rails[0].voltage_min_noise_unaware_uv = + setfllclk->voltuv; + + return 0; +} + +static u32 clk_pmu_vf_inject(struct gk20a *g, struct set_fll_clk *setfllclk) +{ + struct pmu_cmd cmd; + struct pmu_payload payload; + u32 status; + u32 seqdesc; + struct nv_pmu_clk_rpc rpccall; + struct clkrpc_pmucmdhandler_params handler; + + memset(&payload, 0, sizeof(struct pmu_payload)); + memset(&rpccall, 0, sizeof(struct nv_pmu_clk_rpc)); + memset(&handler, 0, sizeof(struct clkrpc_pmucmdhandler_params)); + memset(&cmd, 0, sizeof(struct pmu_cmd)); + + if ((setfllclk->gpc2clkmhz == 0) || (setfllclk->xbar2clkmhz == 0) || + (setfllclk->sys2clkmhz == 0) || (setfllclk->voltuv == 0)) { + return -EINVAL; + } + + if ((setfllclk->target_regime_id_gpc > CTRL_CLK_FLL_REGIME_ID_FR) || + (setfllclk->target_regime_id_sys > CTRL_CLK_FLL_REGIME_ID_FR) || + (setfllclk->target_regime_id_xbar > CTRL_CLK_FLL_REGIME_ID_FR)) { + return -EINVAL; + } + + rpccall.function = NV_PMU_CLK_RPC_ID_CLK_VF_CHANGE_INJECT; + + g->ops.pmu_ver.clk.clk_vf_change_inject_data_fill(g, + &rpccall, setfllclk); + + cmd.hdr.unit_id = PMU_UNIT_CLK; + cmd.hdr.size = (u32)sizeof(struct nv_pmu_clk_cmd) + + (u32)sizeof(struct pmu_hdr); + + cmd.cmd.clk.cmd_type = NV_PMU_CLK_CMD_ID_RPC; + + payload.in.buf = (u8 *)&rpccall; + payload.in.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.in.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.in.offset = NV_PMU_CLK_CMD_RPC_ALLOC_OFFSET; + + payload.out.buf = (u8 *)&rpccall; + payload.out.size = (u32)sizeof(struct nv_pmu_clk_rpc); + payload.out.fb_size = PMU_CMD_SUBMIT_PAYLOAD_PARAMS_FB_SIZE_UNUSED; + payload.out.offset = NV_PMU_CLK_MSG_RPC_ALLOC_OFFSET; + + handler.prpccall = &rpccall; + handler.success = 0; + + status = nvgpu_pmu_cmd_post(g, &cmd, NULL, &payload, + PMU_COMMAND_QUEUE_LPQ, + clkrpc_pmucmdhandler, (void *)&handler, + &seqdesc, ~0); + + if (status) { + nvgpu_err(g, "unable to post clk RPC cmd %x", + cmd.cmd.clk.cmd_type); + goto done; + } + + pmu_wait_message_cond(&g->pmu, + gk20a_get_gr_idle_timeout(g), + &handler.success, 1); + + if (handler.success == 0) { + nvgpu_err(g, "rpc call to inject clock failed"); + status = -EINVAL; + } +done: + return status; +} + +static u32 find_regime_id(struct gk20a *g, u32 domain, u16 clkmhz) +{ + struct fll_device *pflldev; + u8 j; + struct clk_pmupstate *pclk = &g->clk_pmu; + + BOARDOBJGRP_FOR_EACH(&(pclk->avfs_fllobjs.super.super), + struct fll_device *, pflldev, j) { + if (pflldev->clk_domain == domain) { + if (pflldev->regime_desc.fixed_freq_regime_limit_mhz >= + clkmhz) { + return CTRL_CLK_FLL_REGIME_ID_FFR; + } else { + return CTRL_CLK_FLL_REGIME_ID_FR; + } + } + } + return CTRL_CLK_FLL_REGIME_ID_INVALID; +} + +static int set_regime_id(struct gk20a *g, u32 domain, u32 regimeid) +{ + struct fll_device *pflldev; + u8 j; + struct clk_pmupstate *pclk = &g->clk_pmu; + + BOARDOBJGRP_FOR_EACH(&(pclk->avfs_fllobjs.super.super), + struct fll_device *, pflldev, j) { + if (pflldev->clk_domain == domain) { + pflldev->regime_desc.regime_id = regimeid; + return 0; + } + } + return -EINVAL; +} + +static int get_regime_id(struct gk20a *g, u32 domain, u32 *regimeid) +{ + struct fll_device *pflldev; + u8 j; + struct clk_pmupstate *pclk = &g->clk_pmu; + + BOARDOBJGRP_FOR_EACH(&(pclk->avfs_fllobjs.super.super), + struct fll_device *, pflldev, j) { + if (pflldev->clk_domain == domain) { + *regimeid = pflldev->regime_desc.regime_id; + return 0; + } + } + return -EINVAL; +} + +int clk_set_fll_clks(struct gk20a *g, struct set_fll_clk *setfllclk) +{ + int status = -EINVAL; + + /*set regime ids */ + status = get_regime_id(g, CTRL_CLK_DOMAIN_GPC2CLK, + &setfllclk->current_regime_id_gpc); + if (status) { + goto done; + } + + setfllclk->target_regime_id_gpc = find_regime_id(g, + CTRL_CLK_DOMAIN_GPC2CLK, setfllclk->gpc2clkmhz); + + status = get_regime_id(g, CTRL_CLK_DOMAIN_SYS2CLK, + &setfllclk->current_regime_id_sys); + if (status) { + goto done; + } + + setfllclk->target_regime_id_sys = find_regime_id(g, + CTRL_CLK_DOMAIN_SYS2CLK, setfllclk->sys2clkmhz); + + status = get_regime_id(g, CTRL_CLK_DOMAIN_XBAR2CLK, + &setfllclk->current_regime_id_xbar); + if (status) { + goto done; + } + + setfllclk->target_regime_id_xbar = find_regime_id(g, + CTRL_CLK_DOMAIN_XBAR2CLK, setfllclk->xbar2clkmhz); + + status = clk_pmu_vf_inject(g, setfllclk); + + if (status) { + nvgpu_err(g, "vf inject to change clk failed"); + } + + /* save regime ids */ + status = set_regime_id(g, CTRL_CLK_DOMAIN_XBAR2CLK, + setfllclk->target_regime_id_xbar); + if (status) { + goto done; + } + + status = set_regime_id(g, CTRL_CLK_DOMAIN_GPC2CLK, + setfllclk->target_regime_id_gpc); + if (status) { + goto done; + } + + status = set_regime_id(g, CTRL_CLK_DOMAIN_SYS2CLK, + setfllclk->target_regime_id_sys); + if (status) { + goto done; + } +done: + return status; +} + +int clk_get_fll_clks(struct gk20a *g, struct set_fll_clk *setfllclk) +{ + int status = -EINVAL; + struct clk_domain *pdomain; + u8 i; + struct clk_pmupstate *pclk = &g->clk_pmu; + u16 clkmhz = 0; + struct clk_domain_3x_master *p3xmaster; + struct clk_domain_3x_slave *p3xslave; + unsigned long slaveidxmask; + + if (setfllclk->gpc2clkmhz == 0) { + return -EINVAL; + } + + BOARDOBJGRP_FOR_EACH(&(pclk->clk_domainobjs.super.super), + struct clk_domain *, pdomain, i) { + + if (pdomain->api_domain == CTRL_CLK_DOMAIN_GPC2CLK) { + + if (!pdomain->super.implements(g, &pdomain->super, + CTRL_CLK_CLK_DOMAIN_TYPE_3X_MASTER)) { + status = -EINVAL; + goto done; + } + p3xmaster = (struct clk_domain_3x_master *)pdomain; + slaveidxmask = p3xmaster->slave_idxs_mask; + for_each_set_bit(i, &slaveidxmask, 32) { + p3xslave = (struct clk_domain_3x_slave *) + CLK_CLK_DOMAIN_GET(pclk, i); + if ((p3xslave->super.super.super.api_domain != + CTRL_CLK_DOMAIN_XBAR2CLK) && + (p3xslave->super.super.super.api_domain != + CTRL_CLK_DOMAIN_SYS2CLK)) { + continue; + } + clkmhz = 0; + status = p3xslave->clkdomainclkgetslaveclk(g, + pclk, + (struct clk_domain *)p3xslave, + &clkmhz, + setfllclk->gpc2clkmhz); + if (status) { + status = -EINVAL; + goto done; + } + if (p3xslave->super.super.super.api_domain == + CTRL_CLK_DOMAIN_XBAR2CLK) { + setfllclk->xbar2clkmhz = clkmhz; + } + if (p3xslave->super.super.super.api_domain == + CTRL_CLK_DOMAIN_SYS2CLK) { + setfllclk->sys2clkmhz = clkmhz; + } + } + } + } +done: + return status; +} + +u32 clk_domain_print_vf_table(struct gk20a *g, u32 clkapidomain) +{ + u32 status = -EINVAL; + struct clk_domain *pdomain; + u8 i; + struct clk_pmupstate *pclk = &g->clk_pmu; + u16 clkmhz = 0; + u32 volt = 0; + + BOARDOBJGRP_FOR_EACH(&(pclk->clk_domainobjs.super.super), + struct clk_domain *, pdomain, i) { + if (pdomain->api_domain == clkapidomain) { + status = pdomain->clkdomainclkvfsearch(g, pclk, + pdomain, &clkmhz, &volt, + CLK_PROG_VFE_ENTRY_LOGIC); + status = pdomain->clkdomainclkvfsearch(g, pclk, + pdomain, &clkmhz, &volt, + CLK_PROG_VFE_ENTRY_SRAM); + } + } + return status; +} + +static int clk_program_fllclks(struct gk20a *g, struct change_fll_clk *fllclk) +{ + int status = -EINVAL; + struct clk_domain *pdomain; + u8 i; + struct clk_pmupstate *pclk = &g->clk_pmu; + u16 clkmhz = 0; + struct clk_domain_3x_master *p3xmaster; + struct clk_domain_3x_slave *p3xslave; + unsigned long slaveidxmask; + struct set_fll_clk setfllclk; + + if (fllclk->api_clk_domain != CTRL_CLK_DOMAIN_GPCCLK) { + return -EINVAL; + } + if (fllclk->voltuv == 0) { + return -EINVAL; + } + if (fllclk->clkmhz == 0) { + return -EINVAL; + } + + setfllclk.voltuv = fllclk->voltuv; + setfllclk.gpc2clkmhz = fllclk->clkmhz; + + BOARDOBJGRP_FOR_EACH(&(pclk->clk_domainobjs.super.super), + struct clk_domain *, pdomain, i) { + + if (pdomain->api_domain == fllclk->api_clk_domain) { + + if (!pdomain->super.implements(g, &pdomain->super, + CTRL_CLK_CLK_DOMAIN_TYPE_3X_MASTER)) { + status = -EINVAL; + goto done; + } + p3xmaster = (struct clk_domain_3x_master *)pdomain; + slaveidxmask = p3xmaster->slave_idxs_mask; + for_each_set_bit(i, &slaveidxmask, 32) { + p3xslave = (struct clk_domain_3x_slave *) + CLK_CLK_DOMAIN_GET(pclk, i); + if ((p3xslave->super.super.super.api_domain != + CTRL_CLK_DOMAIN_XBARCLK) && + (p3xslave->super.super.super.api_domain != + CTRL_CLK_DOMAIN_SYSCLK)) { + continue; + } + clkmhz = 0; + status = p3xslave->clkdomainclkgetslaveclk(g, + pclk, + (struct clk_domain *)p3xslave, + &clkmhz, + fllclk->clkmhz); + if (status) { + status = -EINVAL; + goto done; + } + if (p3xslave->super.super.super.api_domain == + CTRL_CLK_DOMAIN_XBARCLK) { + setfllclk.xbar2clkmhz = clkmhz; + } + if (p3xslave->super.super.super.api_domain == + CTRL_CLK_DOMAIN_SYSCLK) { + setfllclk.sys2clkmhz = clkmhz; + } + } + } + } + /*set regime ids */ + status = get_regime_id(g, CTRL_CLK_DOMAIN_GPCCLK, + &setfllclk.current_regime_id_gpc); + if (status) { + goto done; + } + + setfllclk.target_regime_id_gpc = find_regime_id(g, + CTRL_CLK_DOMAIN_GPCCLK, setfllclk.gpc2clkmhz); + + status = get_regime_id(g, CTRL_CLK_DOMAIN_SYSCLK, + &setfllclk.current_regime_id_sys); + if (status) { + goto done; + } + + setfllclk.target_regime_id_sys = find_regime_id(g, + CTRL_CLK_DOMAIN_SYSCLK, setfllclk.sys2clkmhz); + + status = get_regime_id(g, CTRL_CLK_DOMAIN_XBARCLK, + &setfllclk.current_regime_id_xbar); + if (status) { + goto done; + } + + setfllclk.target_regime_id_xbar = find_regime_id(g, + CTRL_CLK_DOMAIN_XBARCLK, setfllclk.xbar2clkmhz); + + status = clk_pmu_vf_inject(g, &setfllclk); + + if (status) { + nvgpu_err(g, + "vf inject to change clk failed"); + } + + /* save regime ids */ + status = set_regime_id(g, CTRL_CLK_DOMAIN_XBARCLK, + setfllclk.target_regime_id_xbar); + if (status) { + goto done; + } + + status = set_regime_id(g, CTRL_CLK_DOMAIN_GPCCLK, + setfllclk.target_regime_id_gpc); + if (status) { + goto done; + } + + status = set_regime_id(g, CTRL_CLK_DOMAIN_SYSCLK, + setfllclk.target_regime_id_sys); + if (status) { + goto done; + } +done: + return status; +} + +u32 nvgpu_clk_set_boot_fll_clk_gv10x(struct gk20a *g) +{ + int status; + struct change_fll_clk bootfllclk; + u16 gpcclk_clkmhz = BOOT_GPCCLK_MHZ; + u32 gpcclk_voltuv = 0; + u32 voltuv = 0; + + status = clk_vf_point_cache(g); + if (status) { + nvgpu_err(g,"caching failed"); + return status; + } + + status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPCCLK, + &gpcclk_clkmhz, &gpcclk_voltuv, CTRL_VOLT_DOMAIN_LOGIC); + if (status) { + return status; + } + + voltuv = gpcclk_voltuv; + + status = volt_set_voltage(g, voltuv, 0); + if (status) { + nvgpu_err(g, + "attempt to set boot voltage failed %d", + voltuv); + } + + bootfllclk.api_clk_domain = CTRL_CLK_DOMAIN_GPCCLK; + bootfllclk.clkmhz = gpcclk_clkmhz; + bootfllclk.voltuv = voltuv; + status = clk_program_fllclks(g, &bootfllclk); + if (status) { + nvgpu_err(g, "attempt to set boot gpcclk failed"); + } + + status = clk_pmu_freq_effective_avg_load(g, true); + + /* + * Read clocks after some delay with below method + * & extract clock data from buffer + * clk_freq_effective_avg(g, CTRL_CLK_DOMAIN_GPCCLK | + * CTRL_CLK_DOMAIN_XBARCLK | + * CTRL_CLK_DOMAIN_SYSCLK | + * CTRL_CLK_DOMAIN_NVDCLK) + * */ + + return status; +} + +int nvgpu_clk_set_fll_clk_gv10x(struct gk20a *g) +{ + int status; + struct change_fll_clk bootfllclk; + u16 gpcclk_clkmhz = BOOT_GPCCLK_MHZ; + u32 gpcclk_voltuv = 0U; + u32 voltuv = 0U; + + status = clk_vf_point_cache(g); + if (status != 0) { + nvgpu_err(g, "caching failed"); + return status; + } + + status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPCCLK, + &gpcclk_clkmhz, &gpcclk_voltuv, CTRL_VOLT_DOMAIN_LOGIC); + if (status != 0) { + return status; + } + + voltuv = gpcclk_voltuv; + + status = volt_set_voltage(g, voltuv, 0U); + if (status != 0) { + nvgpu_err(g, "attempt to set max voltage failed %d", voltuv); + } + + bootfllclk.api_clk_domain = CTRL_CLK_DOMAIN_GPCCLK; + bootfllclk.clkmhz = gpcclk_clkmhz; + bootfllclk.voltuv = voltuv; + status = clk_program_fllclks(g, &bootfllclk); + if (status != 0) { + nvgpu_err(g, "attempt to set max gpcclk failed"); + } + return status; +} + +u32 clk_domain_get_f_or_v( + struct gk20a *g, + u32 clkapidomain, + u16 *pclkmhz, + u32 *pvoltuv, + u8 railidx +) +{ + u32 status = -EINVAL; + struct clk_domain *pdomain; + u8 i; + struct clk_pmupstate *pclk = &g->clk_pmu; + u8 rail; + + if ((pclkmhz == NULL) || (pvoltuv == NULL)) { + return -EINVAL; + } + + if (railidx == CTRL_VOLT_DOMAIN_LOGIC) { + rail = CLK_PROG_VFE_ENTRY_LOGIC; + } else if (railidx == CTRL_VOLT_DOMAIN_SRAM) { + rail = CLK_PROG_VFE_ENTRY_SRAM; + } else { + return -EINVAL; + } + + BOARDOBJGRP_FOR_EACH(&(pclk->clk_domainobjs.super.super), + struct clk_domain *, pdomain, i) { + if (pdomain->api_domain == clkapidomain) { + status = pdomain->clkdomainclkvfsearch(g, pclk, + pdomain, pclkmhz, pvoltuv, rail); + return status; + } + } + return status; +} diff --git a/include/clk/clk.h b/include/clk/clk.h new file mode 100644 index 0000000..3f4bdf7 --- /dev/null +++ b/include/clk/clk.h @@ -0,0 +1,144 @@ +/* + * general clock structures & definitions + * + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef NVGPU_CLK_H +#define NVGPU_CLK_H + +#include "clk_vin.h" +#include "clk_fll.h" +#include "clk_domain.h" +#include "clk_prog.h" +#include "clk_vf_point.h" +#include "clk_mclk.h" +#include "clk_freq_controller.h" + +#define NV_PERF_DOMAIN_4X_CLOCK_DOMAIN_SKIP 0x10 +#define NV_PERF_DOMAIN_4X_CLOCK_DOMAIN_MASK 0x1F +#define NV_PERF_DOMAIN_4X_CLOCK_DOMAIN_SHIFT 0 +#define BOOT_GPCCLK_MHZ 952 + +struct gk20a; + +int clk_set_boot_fll_clk(struct gk20a *g); + +/* clock related defines for GPUs supporting clock control from pmu*/ +struct clk_pmupstate { + struct avfsvinobjs avfs_vinobjs; + struct avfsfllobjs avfs_fllobjs; + struct clk_domains clk_domainobjs; + struct clk_progs clk_progobjs; + struct clk_vf_points clk_vf_pointobjs; + struct clk_mclk_state clk_mclk; + struct clk_freq_controllers clk_freq_controllers; +}; + +struct clockentry { + u8 vbios_clk_domain; + u8 clk_which; + u8 perf_index; + u32 api_clk_domain; +}; + +struct change_fll_clk { + u32 api_clk_domain; + u16 clkmhz; + u32 voltuv; +}; + +struct set_fll_clk { + u32 voltuv; + u16 gpc2clkmhz; + u32 current_regime_id_gpc; + u32 target_regime_id_gpc; + u16 sys2clkmhz; + u32 current_regime_id_sys; + u32 target_regime_id_sys; + u16 xbar2clkmhz; + u32 current_regime_id_xbar; + u32 target_regime_id_xbar; +}; + +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_MAX_NUMCLKS 9 + +struct vbios_clock_domain { + u8 clock_type; + u8 num_domains; + struct clockentry clock_entry[NV_PERF_HEADER_4X_CLOCKS_DOMAINS_MAX_NUMCLKS]; +}; + +struct vbios_clocks_table_1x_hal_clock_entry { + enum nv_pmu_clk_clkwhich domain; + bool b_noise_aware_capable; + u8 clk_vf_curve_count; +}; + +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_4_GPC2CLK 0 +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_4_XBAR2CLK 1 +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_4_DRAMCLK 2 +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_4_SYS2CLK 3 +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_4_HUB2CLK 4 +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_4_MSDCLK 5 +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_4_PWRCLK 6 +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_4_DISPCLK 7 +#define NV_PERF_HEADER_4X_CLOCKS_DOMAINS_4_NUMCLKS 8 + +#define PERF_CLK_MCLK 0 +#define PERF_CLK_DISPCLK 1 +#define PERF_CLK_GPC2CLK 2 +#define PERF_CLK_HOSTCLK 3 +#define PERF_CLK_LTC2CLK 4 +#define PERF_CLK_SYS2CLK 5 +#define PERF_CLK_HUB2CLK 6 +#define PERF_CLK_LEGCLK 7 +#define PERF_CLK_MSDCLK 8 +#define PERF_CLK_XCLK 9 +#define PERF_CLK_PWRCLK 10 +#define PERF_CLK_XBAR2CLK 11 +#define PERF_CLK_PCIEGENCLK 12 +#define PERF_CLK_NUM 13 + +#define BOOT_GPC2CLK_MHZ 2581 + +u32 clk_pmu_vin_load(struct gk20a *g); +u32 clk_domain_print_vf_table(struct gk20a *g, u32 clkapidomain); +u32 clk_domain_get_f_or_v( + struct gk20a *g, + u32 clkapidomain, + u16 *pclkmhz, + u32 *pvoltuv, + u8 railidx +); +int clk_get_fll_clks(struct gk20a *g, struct set_fll_clk *fllclk); +int clk_set_fll_clks(struct gk20a *g, struct set_fll_clk *fllclk); +int clk_pmu_freq_controller_load(struct gk20a *g, bool bload, u8 bit_idx); +u32 nvgpu_clk_vf_change_inject_data_fill_gv10x(struct gk20a *g, + struct nv_pmu_clk_rpc *rpccall, + struct set_fll_clk *setfllclk); +u32 nvgpu_clk_vf_change_inject_data_fill_gp10x(struct gk20a *g, + struct nv_pmu_clk_rpc *rpccall, + struct set_fll_clk *setfllclk); +u32 nvgpu_clk_set_boot_fll_clk_gv10x(struct gk20a *g); +int nvgpu_clk_set_fll_clk_gv10x(struct gk20a *g); +int clk_pmu_freq_effective_avg_load(struct gk20a *g, bool bload); +u32 clk_freq_effective_avg(struct gk20a *g, u32 clkDomainMask); +#endif /* NVGPU_CLK_H */ diff --git a/include/clk/clk_arb.c b/include/clk/clk_arb.c new file mode 100644 index 0000000..6cf005c --- /dev/null +++ b/include/clk/clk_arb.c @@ -0,0 +1,1087 @@ +/* + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clk/clk.h" +#include "pstate/pstate.h" +#include "lpwr/lpwr.h" +#include "volt/volt.h" + +int nvgpu_clk_notification_queue_alloc(struct gk20a *g, + struct nvgpu_clk_notification_queue *queue, + size_t events_number) { + queue->notifications = nvgpu_kcalloc(g, events_number, + sizeof(struct nvgpu_clk_notification)); + if (!queue->notifications) + return -ENOMEM; + queue->size = events_number; + + nvgpu_atomic_set(&queue->head, 0); + nvgpu_atomic_set(&queue->tail, 0); + + return 0; +} + +void nvgpu_clk_notification_queue_free(struct gk20a *g, + struct nvgpu_clk_notification_queue *queue) { + if (queue->size > 0) { + nvgpu_kfree(g, queue->notifications); + queue->size = 0; + nvgpu_atomic_set(&queue->head, 0); + nvgpu_atomic_set(&queue->tail, 0); + } +} + +static void nvgpu_clk_arb_queue_notification(struct gk20a *g, + struct nvgpu_clk_notification_queue *queue, + u32 alarm_mask) { + + u32 queue_index; + u64 timestamp; + + queue_index = (nvgpu_atomic_inc_return(&queue->tail)) % queue->size; + /* get current timestamp */ + timestamp = (u64) nvgpu_hr_timestamp(); + + queue->notifications[queue_index].timestamp = timestamp; + queue->notifications[queue_index].notification = alarm_mask; + +} + +void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm) +{ + struct nvgpu_clk_arb *arb = g->clk_arb; + + u64 current_mask; + u32 refcnt; + u32 alarm_mask; + u64 new_mask; + + do { + current_mask = nvgpu_atomic64_read(&arb->alarm_mask); + /* atomic operations are strong so they do not need masks */ + + refcnt = ((u32) (current_mask >> 32)) + 1; + alarm_mask = (u32) (current_mask & ~0) | alarm; + new_mask = ((u64) refcnt << 32) | alarm_mask; + + } while (unlikely(current_mask != + (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask, + current_mask, new_mask))); + + nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm); +} + + +int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb) +{ + struct gk20a *g = arb->g; + struct nvgpu_clk_vf_table *table; + + u32 i, j; + int status = -EINVAL; + u32 gpc2clk_voltuv = 0, mclk_voltuv = 0; + u32 gpc2clk_voltuv_sram = 0, mclk_voltuv_sram = 0; + u16 clk_cur; + u32 num_points; + + struct clk_set_info *p5_info, *p0_info; + + table = NV_ACCESS_ONCE(arb->current_vf_table); + /* make flag visible when all data has resolved in the tables */ + nvgpu_smp_rmb(); + + table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] : + &arb->vf_table_pool[0]; + + /* Get allowed memory ranges */ + if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_GPC2CLK, + &arb->gpc2clk_min, + &arb->gpc2clk_max) < 0) { + nvgpu_err(g, "failed to fetch GPC2CLK range"); + goto exit_vf_table; + } + if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_MCLK, + &arb->mclk_min, + &arb->mclk_max) < 0) { + nvgpu_err(g, "failed to fetch MCLK range"); + goto exit_vf_table; + } + + table->gpc2clk_num_points = MAX_F_POINTS; + table->mclk_num_points = MAX_F_POINTS; + + if (g->ops.clk.clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_GPC2CLK, + &table->gpc2clk_num_points, arb->gpc2clk_f_points)) { + nvgpu_err(g, "failed to fetch GPC2CLK frequency points"); + goto exit_vf_table; + } + + if (g->ops.clk.clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_MCLK, + &table->mclk_num_points, arb->mclk_f_points)) { + nvgpu_err(g, "failed to fetch MCLK frequency points"); + goto exit_vf_table; + } + if (!table->mclk_num_points || !table->gpc2clk_num_points) { + nvgpu_err(g, "empty queries to f points mclk %d gpc2clk %d", + table->mclk_num_points, table->gpc2clk_num_points); + status = -EINVAL; + goto exit_vf_table; + } + + memset(table->mclk_points, 0, + table->mclk_num_points*sizeof(struct nvgpu_clk_vf_point)); + memset(table->gpc2clk_points, 0, + table->gpc2clk_num_points*sizeof(struct nvgpu_clk_vf_point)); + + p5_info = pstate_get_clk_set_info(g, + CTRL_PERF_PSTATE_P5, clkwhich_mclk); + if (!p5_info) { + nvgpu_err(g, "failed to get MCLK P5 info"); + goto exit_vf_table; + } + p0_info = pstate_get_clk_set_info(g, + CTRL_PERF_PSTATE_P0, clkwhich_mclk); + if (!p0_info) { + nvgpu_err(g, "failed to get MCLK P0 info"); + goto exit_vf_table; + } + + for (i = 0, j = 0, num_points = 0, clk_cur = 0; + i < table->mclk_num_points; i++) { + + if ((arb->mclk_f_points[i] >= arb->mclk_min) && + (arb->mclk_f_points[i] <= arb->mclk_max) && + (arb->mclk_f_points[i] != clk_cur)) { + + table->mclk_points[j].mem_mhz = arb->mclk_f_points[i]; + mclk_voltuv = mclk_voltuv_sram = 0; + + status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK, + &table->mclk_points[j].mem_mhz, &mclk_voltuv, + CTRL_VOLT_DOMAIN_LOGIC); + if (status < 0) { + nvgpu_err(g, + "failed to get MCLK LOGIC voltage"); + goto exit_vf_table; + } + status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_MCLK, + &table->mclk_points[j].mem_mhz, + &mclk_voltuv_sram, + CTRL_VOLT_DOMAIN_SRAM); + if (status < 0) { + nvgpu_err(g, "failed to get MCLK SRAM voltage"); + goto exit_vf_table; + } + + table->mclk_points[j].uvolt = mclk_voltuv; + table->mclk_points[j].uvolt_sram = mclk_voltuv_sram; + clk_cur = table->mclk_points[j].mem_mhz; + + if ((clk_cur >= p5_info->min_mhz) && + (clk_cur <= p5_info->max_mhz)) + VF_POINT_SET_PSTATE_SUPPORTED( + &table->mclk_points[j], + CTRL_PERF_PSTATE_P5); + if ((clk_cur >= p0_info->min_mhz) && + (clk_cur <= p0_info->max_mhz)) + VF_POINT_SET_PSTATE_SUPPORTED( + &table->mclk_points[j], + CTRL_PERF_PSTATE_P0); + + j++; + num_points++; + + } + } + table->mclk_num_points = num_points; + + p5_info = pstate_get_clk_set_info(g, + CTRL_PERF_PSTATE_P5, clkwhich_gpc2clk); + if (!p5_info) { + status = -EINVAL; + nvgpu_err(g, "failed to get GPC2CLK P5 info"); + goto exit_vf_table; + } + + p0_info = pstate_get_clk_set_info(g, + CTRL_PERF_PSTATE_P0, clkwhich_gpc2clk); + if (!p0_info) { + status = -EINVAL; + nvgpu_err(g, "failed to get GPC2CLK P0 info"); + goto exit_vf_table; + } + + /* GPC2CLK needs to be checked in two passes. The first determines the + * relationships between GPC2CLK, SYS2CLK and XBAR2CLK, while the + * second verifies that the clocks minimum is satisfied and sets + * the voltages + */ + for (i = 0, j = 0, num_points = 0, clk_cur = 0; + i < table->gpc2clk_num_points; i++) { + struct set_fll_clk setfllclk; + + if ((arb->gpc2clk_f_points[i] >= arb->gpc2clk_min) && + (arb->gpc2clk_f_points[i] <= arb->gpc2clk_max) && + (arb->gpc2clk_f_points[i] != clk_cur)) { + + table->gpc2clk_points[j].gpc_mhz = + arb->gpc2clk_f_points[i]; + setfllclk.gpc2clkmhz = arb->gpc2clk_f_points[i]; + status = clk_get_fll_clks(g, &setfllclk); + if (status < 0) { + nvgpu_err(g, + "failed to get GPC2CLK slave clocks"); + goto exit_vf_table; + } + + table->gpc2clk_points[j].sys_mhz = + setfllclk.sys2clkmhz; + table->gpc2clk_points[j].xbar_mhz = + setfllclk.xbar2clkmhz; + + clk_cur = table->gpc2clk_points[j].gpc_mhz; + + if ((clk_cur >= p5_info->min_mhz) && + (clk_cur <= p5_info->max_mhz)) + VF_POINT_SET_PSTATE_SUPPORTED( + &table->gpc2clk_points[j], + CTRL_PERF_PSTATE_P5); + if ((clk_cur >= p0_info->min_mhz) && + (clk_cur <= p0_info->max_mhz)) + VF_POINT_SET_PSTATE_SUPPORTED( + &table->gpc2clk_points[j], + CTRL_PERF_PSTATE_P0); + + j++; + num_points++; + } + } + table->gpc2clk_num_points = num_points; + + /* Second pass */ + for (i = 0, j = 0; i < table->gpc2clk_num_points; i++) { + + u16 alt_gpc2clk = table->gpc2clk_points[i].gpc_mhz; + + gpc2clk_voltuv = gpc2clk_voltuv_sram = 0; + + /* Check sysclk */ + p5_info = pstate_get_clk_set_info(g, + VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]), + clkwhich_sys2clk); + if (!p5_info) { + status = -EINVAL; + nvgpu_err(g, "failed to get SYS2CLK P5 info"); + goto exit_vf_table; + } + + /* sys2clk below clk min, need to find correct clock */ + if (table->gpc2clk_points[i].sys_mhz < p5_info->min_mhz) { + for (j = i + 1; j < table->gpc2clk_num_points; j++) { + + if (table->gpc2clk_points[j].sys_mhz >= + p5_info->min_mhz) { + + + table->gpc2clk_points[i].sys_mhz = + p5_info->min_mhz; + + alt_gpc2clk = alt_gpc2clk < + table->gpc2clk_points[j]. + gpc_mhz ? + table->gpc2clk_points[j]. + gpc_mhz : + alt_gpc2clk; + break; + } + } + /* no VF exists that satisfies condition */ + if (j == table->gpc2clk_num_points) { + nvgpu_err(g, "NO SYS2CLK VF point possible"); + status = -EINVAL; + goto exit_vf_table; + } + } + + /* Check xbarclk */ + p5_info = pstate_get_clk_set_info(g, + VF_POINT_GET_PSTATE(&table->gpc2clk_points[i]), + clkwhich_xbar2clk); + if (!p5_info) { + status = -EINVAL; + nvgpu_err(g, "failed to get SYS2CLK P5 info"); + goto exit_vf_table; + } + + /* xbar2clk below clk min, need to find correct clock */ + if (table->gpc2clk_points[i].xbar_mhz < p5_info->min_mhz) { + for (j = i; j < table->gpc2clk_num_points; j++) { + if (table->gpc2clk_points[j].xbar_mhz >= + p5_info->min_mhz) { + + table->gpc2clk_points[i].xbar_mhz = + p5_info->min_mhz; + + alt_gpc2clk = alt_gpc2clk < + table->gpc2clk_points[j]. + gpc_mhz ? + table->gpc2clk_points[j]. + gpc_mhz : + alt_gpc2clk; + break; + } + } + /* no VF exists that satisfies condition */ + if (j == table->gpc2clk_num_points) { + status = -EINVAL; + nvgpu_err(g, "NO XBAR2CLK VF point possible"); + + goto exit_vf_table; + } + } + + /* Calculate voltages */ + status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK, + &alt_gpc2clk, &gpc2clk_voltuv, + CTRL_VOLT_DOMAIN_LOGIC); + if (status < 0) { + nvgpu_err(g, "failed to get GPC2CLK LOGIC voltage"); + goto exit_vf_table; + } + + status = clk_domain_get_f_or_v(g, CTRL_CLK_DOMAIN_GPC2CLK, + &alt_gpc2clk, + &gpc2clk_voltuv_sram, + CTRL_VOLT_DOMAIN_SRAM); + if (status < 0) { + nvgpu_err(g, "failed to get GPC2CLK SRAM voltage"); + goto exit_vf_table; + } + + table->gpc2clk_points[i].uvolt = gpc2clk_voltuv; + table->gpc2clk_points[i].uvolt_sram = gpc2clk_voltuv_sram; + } + + /* make table visible when all data has resolved in the tables */ + nvgpu_smp_wmb(); + arb->current_vf_table = table; + +exit_vf_table: + + if (status < 0) + nvgpu_clk_arb_set_global_alarm(g, + EVENT(ALARM_VF_TABLE_UPDATE_FAILED)); + nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item); + + return status; +} + + +static void nvgpu_clk_arb_run_vf_table_cb(struct nvgpu_clk_arb *arb) +{ + struct gk20a *g = arb->g; + u32 err; + + /* get latest vf curve from pmu */ + err = clk_vf_point_cache(g); + if (err) { + nvgpu_err(g, "failed to cache VF table"); + nvgpu_clk_arb_set_global_alarm(g, + EVENT(ALARM_VF_TABLE_UPDATE_FAILED)); + nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item); + + return; + } + nvgpu_clk_arb_update_vf_table(arb); +} + +u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, + struct nvgpu_clk_arb_target *target, + u32 alarm) { + + struct nvgpu_clk_session *session = dev->session; + struct nvgpu_clk_arb *arb = session->g->clk_arb; + struct nvgpu_clk_notification *notification; + + u32 queue_alarm_mask = 0; + u32 enabled_mask = 0; + u32 new_alarms_reported = 0; + u32 poll_mask = 0; + u32 tail, head; + u32 queue_index; + size_t size; + int index; + + enabled_mask = nvgpu_atomic_read(&dev->enabled_mask); + size = arb->notification_queue.size; + + /* queue global arbiter notifications in buffer */ + do { + tail = nvgpu_atomic_read(&arb->notification_queue.tail); + /* copy items to the queue */ + queue_index = nvgpu_atomic_read(&dev->queue.tail); + head = dev->arb_queue_head; + head = (tail - head) < arb->notification_queue.size ? + head : tail - arb->notification_queue.size; + + for (index = head; _WRAPGTEQ(tail, index); index++) { + u32 alarm_detected; + + notification = &arb->notification_queue. + notifications[(index+1) % size]; + alarm_detected = + NV_ACCESS_ONCE(notification->notification); + + if (!(enabled_mask & alarm_detected)) + continue; + + queue_index++; + dev->queue.notifications[ + queue_index % dev->queue.size].timestamp = + NV_ACCESS_ONCE(notification->timestamp); + + dev->queue.notifications[ + queue_index % dev->queue.size].notification = + alarm_detected; + + queue_alarm_mask |= alarm_detected; + } + } while (unlikely(nvgpu_atomic_read(&arb->notification_queue.tail) != + (int)tail)); + + nvgpu_atomic_set(&dev->queue.tail, queue_index); + /* update the last notification we processed from global queue */ + + dev->arb_queue_head = tail; + + /* Check if current session targets are met */ + if (enabled_mask & EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)) { + if ((target->gpc2clk < session->target->gpc2clk) + || (target->mclk < session->target->mclk)) { + + poll_mask |= (NVGPU_POLLIN | NVGPU_POLLPRI); + nvgpu_clk_arb_queue_notification(arb->g, &dev->queue, + EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)); + } + } + + /* Check if there is a new VF update */ + if (queue_alarm_mask & EVENT(VF_UPDATE)) + poll_mask |= (NVGPU_POLLIN | NVGPU_POLLRDNORM); + + /* Notify sticky alarms that were not reported on previous run*/ + new_alarms_reported = (queue_alarm_mask | + (alarm & ~dev->alarms_reported & queue_alarm_mask)); + + if (new_alarms_reported & ~LOCAL_ALARM_MASK) { + /* check that we are not re-reporting */ + if (new_alarms_reported & EVENT(ALARM_GPU_LOST)) + poll_mask |= NVGPU_POLLHUP; + + poll_mask |= (NVGPU_POLLIN | NVGPU_POLLPRI); + /* On next run do not report global alarms that were already + * reported, but report SHUTDOWN always + */ + dev->alarms_reported = new_alarms_reported & ~LOCAL_ALARM_MASK & + ~EVENT(ALARM_GPU_LOST); + } + + if (poll_mask) { + nvgpu_atomic_set(&dev->poll_mask, poll_mask); + nvgpu_clk_arb_event_post_event(dev); + } + + return new_alarms_reported; +} + +void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm) +{ + struct nvgpu_clk_arb *arb = g->clk_arb; + + u64 current_mask; + u32 refcnt; + u32 alarm_mask; + u64 new_mask; + + do { + current_mask = nvgpu_atomic64_read(&arb->alarm_mask); + /* atomic operations are strong so they do not need masks */ + + refcnt = ((u32) (current_mask >> 32)) + 1; + alarm_mask = (u32) (current_mask & ~alarm); + new_mask = ((u64) refcnt << 32) | alarm_mask; + + } while (unlikely(current_mask != + (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask, + current_mask, new_mask))); +} + +/* + * Process one scheduled work item. + */ +static void nvgpu_clk_arb_worker_process_item( + struct nvgpu_clk_arb_work_item *work_item) +{ + struct gk20a *g = work_item->arb->g; + + clk_arb_dbg(g, " "); + + if (work_item->item_type == CLK_ARB_WORK_UPDATE_VF_TABLE) + nvgpu_clk_arb_run_vf_table_cb(work_item->arb); + else if (work_item->item_type == CLK_ARB_WORK_UPDATE_ARB) + g->ops.clk_arb.clk_arb_run_arbiter_cb(work_item->arb); +} + +/** + * Tell the worker that one more work needs to be done. + * + * Increase the work counter to synchronize the worker with the new work. Wake + * up the worker. If the worker was already running, it will handle this work + * before going to sleep. + */ +static int nvgpu_clk_arb_worker_wakeup(struct gk20a *g) +{ + int put; + + clk_arb_dbg(g, " "); + + put = nvgpu_atomic_inc_return(&g->clk_arb_worker.put); + nvgpu_cond_signal_interruptible(&g->clk_arb_worker.wq); + + return put; +} + +/** + * Test if there is some work pending. + * + * This is a pair for nvgpu_clk_arb_worker_wakeup to be called from the + * worker. The worker has an internal work counter which is incremented once + * per finished work item. This is compared with the number of queued jobs. + */ +static bool nvgpu_clk_arb_worker_pending(struct gk20a *g, int get) +{ + bool pending = nvgpu_atomic_read(&g->clk_arb_worker.put) != get; + + /* We don't need barriers because they are implicit in locking */ + return pending; +} + +/** + * Process the queued works for the worker thread serially. + * + * Flush all the work items in the queue one by one. This may block timeout + * handling for a short while, as these are serialized. + */ +static void nvgpu_clk_arb_worker_process(struct gk20a *g, int *get) +{ + + while (nvgpu_clk_arb_worker_pending(g, *get)) { + struct nvgpu_clk_arb_work_item *work_item = NULL; + + nvgpu_spinlock_acquire(&g->clk_arb_worker.items_lock); + if (!nvgpu_list_empty(&g->clk_arb_worker.items)) { + work_item = nvgpu_list_first_entry(&g->clk_arb_worker.items, + nvgpu_clk_arb_work_item, worker_item); + nvgpu_list_del(&work_item->worker_item); + } + nvgpu_spinlock_release(&g->clk_arb_worker.items_lock); + + if (!work_item) { + /* + * Woke up for some other reason, but there are no + * other reasons than a work item added in the items list + * currently, so warn and ack the message. + */ + nvgpu_warn(g, "Spurious worker event!"); + ++*get; + break; + } + + nvgpu_clk_arb_worker_process_item(work_item); + ++*get; + } +} + +/* + * Process all work items found in the clk arbiter work queue. + */ +static int nvgpu_clk_arb_poll_worker(void *arg) +{ + struct gk20a *g = (struct gk20a *)arg; + struct gk20a_worker *worker = &g->clk_arb_worker; + int get = 0; + + clk_arb_dbg(g, " "); + + while (!nvgpu_thread_should_stop(&worker->poll_task)) { + int ret; + + ret = NVGPU_COND_WAIT_INTERRUPTIBLE( + &worker->wq, + nvgpu_clk_arb_worker_pending(g, get), 0); + + if (nvgpu_thread_should_stop(&worker->poll_task)) { + break; + } + + if (ret == 0) + nvgpu_clk_arb_worker_process(g, &get); + } + return 0; +} + +static int __nvgpu_clk_arb_worker_start(struct gk20a *g) +{ + char thread_name[64]; + int err = 0; + + if (nvgpu_thread_is_running(&g->clk_arb_worker.poll_task)) + return err; + + nvgpu_mutex_acquire(&g->clk_arb_worker.start_lock); + + /* + * Mutexes have implicit barriers, so there is no risk of a thread + * having a stale copy of the poll_task variable as the call to + * thread_is_running is volatile + */ + + if (nvgpu_thread_is_running(&g->clk_arb_worker.poll_task)) { + nvgpu_mutex_release(&g->clk_arb_worker.start_lock); + return err; + } + + snprintf(thread_name, sizeof(thread_name), + "nvgpu_clk_arb_poll_%s", g->name); + + err = nvgpu_thread_create(&g->clk_arb_worker.poll_task, g, + nvgpu_clk_arb_poll_worker, thread_name); + + nvgpu_mutex_release(&g->clk_arb_worker.start_lock); + return err; +} + +/** + * Append a work item to the worker's list. + * + * This adds work item to the end of the list and wakes the worker + * up immediately. If the work item already existed in the list, it's not added, + * because in that case it has been scheduled already but has not yet been + * processed. + */ +void nvgpu_clk_arb_worker_enqueue(struct gk20a *g, + struct nvgpu_clk_arb_work_item *work_item) +{ + clk_arb_dbg(g, " "); + + /* + * Warn if worker thread cannot run + */ + if (WARN_ON(__nvgpu_clk_arb_worker_start(g))) { + nvgpu_warn(g, "clk arb worker cannot run!"); + return; + } + + nvgpu_spinlock_acquire(&g->clk_arb_worker.items_lock); + if (!nvgpu_list_empty(&work_item->worker_item)) { + /* + * Already queued, so will get processed eventually. + * The worker is probably awake already. + */ + nvgpu_spinlock_release(&g->clk_arb_worker.items_lock); + return; + } + nvgpu_list_add_tail(&work_item->worker_item, &g->clk_arb_worker.items); + nvgpu_spinlock_release(&g->clk_arb_worker.items_lock); + + nvgpu_clk_arb_worker_wakeup(g); +} + +/** + * Initialize the clk arb worker's metadata and start the background thread. + */ +int nvgpu_clk_arb_worker_init(struct gk20a *g) +{ + int err; + + nvgpu_atomic_set(&g->clk_arb_worker.put, 0); + nvgpu_cond_init(&g->clk_arb_worker.wq); + nvgpu_init_list_node(&g->clk_arb_worker.items); + nvgpu_spinlock_init(&g->clk_arb_worker.items_lock); + err = nvgpu_mutex_init(&g->clk_arb_worker.start_lock); + if (err) + goto error_check; + + err = __nvgpu_clk_arb_worker_start(g); +error_check: + if (err) { + nvgpu_err(g, "failed to start clk arb poller thread"); + return err; + } + return 0; +} + +int nvgpu_clk_arb_init_arbiter(struct gk20a *g) +{ + int err = 0; + + if (!g->ops.clk.support_clk_freq_controller || + !g->ops.clk_arb.get_arbiter_clk_domains) { + return 0; + } + + nvgpu_mutex_acquire(&g->clk_arb_enable_lock); + + err = g->ops.clk_arb.arbiter_clk_init(g); + + nvgpu_mutex_release(&g->clk_arb_enable_lock); + + return err; +} + +bool nvgpu_clk_arb_has_active_req(struct gk20a *g) +{ + return (nvgpu_atomic_read(&g->clk_arb_global_nr) > 0); +} + +void nvgpu_clk_arb_send_thermal_alarm(struct gk20a *g) +{ + nvgpu_clk_arb_schedule_alarm(g, + (0x1UL << NVGPU_EVENT_ALARM_THERMAL_ABOVE_THRESHOLD)); +} + +void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm) +{ + struct nvgpu_clk_arb *arb = g->clk_arb; + + nvgpu_clk_arb_set_global_alarm(g, alarm); + nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item); +} + +static void nvgpu_clk_arb_worker_deinit(struct gk20a *g) +{ + nvgpu_atomic_inc(&g->clk_arb_worker.put); + + nvgpu_mutex_acquire(&g->clk_arb_worker.start_lock); + nvgpu_thread_stop(&g->clk_arb_worker.poll_task); + nvgpu_mutex_release(&g->clk_arb_worker.start_lock); +} + +void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g) +{ + struct nvgpu_clk_arb *arb = g->clk_arb; + + nvgpu_mutex_acquire(&g->clk_arb_enable_lock); + + if (arb) { + nvgpu_clk_arb_worker_deinit(g); + g->ops.clk_arb.clk_arb_cleanup(g->clk_arb); + } + + nvgpu_mutex_release(&g->clk_arb_enable_lock); +} + +int nvgpu_clk_arb_init_session(struct gk20a *g, + struct nvgpu_clk_session **_session) +{ + struct nvgpu_clk_arb *arb = g->clk_arb; + struct nvgpu_clk_session *session = *(_session); + + clk_arb_dbg(g, " "); + + if (!g->ops.clk.support_clk_freq_controller || + !g->ops.clk_arb.get_arbiter_clk_domains) { + return 0; + } + + session = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_session)); + if (!session) + return -ENOMEM; + session->g = g; + + nvgpu_ref_init(&session->refcount); + + session->zombie = false; + session->target_pool[0].pstate = CTRL_PERF_PSTATE_P8; + /* make sure that the initialization of the pool is visible + * before the update + */ + nvgpu_smp_wmb(); + session->target = &session->target_pool[0]; + + nvgpu_init_list_node(&session->targets); + nvgpu_spinlock_init(&session->session_lock); + + nvgpu_spinlock_acquire(&arb->sessions_lock); + nvgpu_list_add_tail(&session->link, &arb->sessions); + nvgpu_spinlock_release(&arb->sessions_lock); + + *_session = session; + + return 0; +} + +void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount) +{ + struct nvgpu_clk_dev *dev = container_of(refcount, + struct nvgpu_clk_dev, refcount); + struct nvgpu_clk_session *session = dev->session; + struct gk20a *g = session->g; + + nvgpu_clk_notification_queue_free(g, &dev->queue); + + nvgpu_atomic_dec(&g->clk_arb_global_nr); + nvgpu_kfree(g, dev); +} + +void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount) +{ + struct nvgpu_clk_session *session = container_of(refcount, + struct nvgpu_clk_session, refcount); + struct nvgpu_clk_arb *arb = session->g->clk_arb; + struct gk20a *g = session->g; + struct nvgpu_clk_dev *dev, *tmp; + + clk_arb_dbg(g, " "); + + if (arb) { + nvgpu_spinlock_acquire(&arb->sessions_lock); + nvgpu_list_del(&session->link); + nvgpu_spinlock_release(&arb->sessions_lock); + } + + nvgpu_spinlock_acquire(&session->session_lock); + nvgpu_list_for_each_entry_safe(dev, tmp, &session->targets, + nvgpu_clk_dev, node) { + nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd); + nvgpu_list_del(&dev->node); + } + nvgpu_spinlock_release(&session->session_lock); + + nvgpu_kfree(g, session); +} + +void nvgpu_clk_arb_release_session(struct gk20a *g, + struct nvgpu_clk_session *session) +{ + struct nvgpu_clk_arb *arb = g->clk_arb; + + clk_arb_dbg(g, " "); + + session->zombie = true; + nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session); + if (arb) + nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item); +} + +void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g) +{ + struct nvgpu_clk_arb *arb = g->clk_arb; + + nvgpu_clk_arb_worker_enqueue(g, &arb->update_vf_table_work_item); +} + +/* This function is inherently unsafe to call while arbiter is running + * arbiter must be blocked before calling this function + */ +int nvgpu_clk_arb_get_current_pstate(struct gk20a *g) +{ + return NV_ACCESS_ONCE(g->clk_arb->actual->pstate); +} + +void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock) +{ + struct nvgpu_clk_arb *arb = g->clk_arb; + + if (lock) + nvgpu_mutex_acquire(&arb->pstate_lock); + else + nvgpu_mutex_release(&arb->pstate_lock); +} + +bool nvgpu_clk_arb_is_valid_domain(struct gk20a *g, u32 api_domain) +{ + u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g); + + switch (api_domain) { + case NVGPU_CLK_DOMAIN_MCLK: + return (clk_domains & CTRL_CLK_DOMAIN_MCLK) != 0; + + case NVGPU_CLK_DOMAIN_GPCCLK: + return (clk_domains & CTRL_CLK_DOMAIN_GPC2CLK) != 0; + + default: + return false; + } +} + +int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain, + u16 *min_mhz, u16 *max_mhz) +{ + int ret; + + switch (api_domain) { + case NVGPU_CLK_DOMAIN_MCLK: + ret = g->ops.clk_arb.get_arbiter_clk_range(g, + CTRL_CLK_DOMAIN_MCLK, min_mhz, max_mhz); + return ret; + + case NVGPU_CLK_DOMAIN_GPCCLK: + ret = g->ops.clk_arb.get_arbiter_clk_range(g, + CTRL_CLK_DOMAIN_GPC2CLK, min_mhz, max_mhz); + if (!ret) { + *min_mhz /= 2; + *max_mhz /= 2; + } + return ret; + + default: + return -EINVAL; + } +} + +int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g, + u32 api_domain, u32 *max_points, u16 *fpoints) +{ + int err; + u32 i; + + switch (api_domain) { + case NVGPU_CLK_DOMAIN_GPCCLK: + err = g->ops.clk_arb.get_arbiter_f_points(g, + CTRL_CLK_DOMAIN_GPC2CLK, max_points, fpoints); + if (err || !fpoints) + return err; + for (i = 0; i < *max_points; i++) + fpoints[i] /= 2; + return 0; + case NVGPU_CLK_DOMAIN_MCLK: + return g->ops.clk_arb.get_arbiter_f_points(g, + CTRL_CLK_DOMAIN_MCLK, max_points, fpoints); + default: + return -EINVAL; + } +} + +int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session, + u32 api_domain, u16 *freq_mhz) +{ + int err = 0; + struct nvgpu_clk_arb_target *target = session->target; + + if (!nvgpu_clk_arb_is_valid_domain(session->g, api_domain)) { + return -EINVAL; + } + + switch (api_domain) { + case NVGPU_CLK_DOMAIN_MCLK: + *freq_mhz = target->mclk; + break; + + case NVGPU_CLK_DOMAIN_GPCCLK: + *freq_mhz = target->gpc2clk / 2ULL; + break; + + default: + *freq_mhz = 0; + err = -EINVAL; + } + return err; +} + +int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g, + u32 api_domain, u16 *freq_mhz) +{ + struct nvgpu_clk_arb *arb = g->clk_arb; + int err = 0; + struct nvgpu_clk_arb_target *actual = arb->actual; + + if (!nvgpu_clk_arb_is_valid_domain(g, api_domain)) { + return -EINVAL; + } + + switch (api_domain) { + case NVGPU_CLK_DOMAIN_MCLK: + *freq_mhz = actual->mclk; + break; + + case NVGPU_CLK_DOMAIN_GPCCLK: + *freq_mhz = actual->gpc2clk / 2ULL; + break; + + default: + *freq_mhz = 0; + err = -EINVAL; + } + return err; +} + +unsigned long nvgpu_clk_measure_freq(struct gk20a *g, u32 api_domain) +{ + unsigned long freq = 0UL; + + switch (api_domain) { + case CTRL_CLK_DOMAIN_GPC2CLK: + freq = g->ops.clk.get_rate(g, CTRL_CLK_DOMAIN_GPCCLK) * 2UL; + break; + default: + break; + } + return freq; +} + +int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g, + u32 api_domain, u16 *freq_mhz) +{ + if (!nvgpu_clk_arb_is_valid_domain(g, api_domain)) { + return -EINVAL; + } + + switch (api_domain) { + case NVGPU_CLK_DOMAIN_MCLK: + *freq_mhz = g->ops.clk.measure_freq(g, CTRL_CLK_DOMAIN_MCLK) / + 1000000ULL; + return 0; + + case NVGPU_CLK_DOMAIN_GPCCLK: + *freq_mhz = g->ops.clk.measure_freq(g, + CTRL_CLK_DOMAIN_GPC2CLK) / 2000000ULL; + return 0; + + default: + return -EINVAL; + } +} diff --git a/include/clk/clk_domain.c b/include/clk/clk_domain.c new file mode 100644 index 0000000..3b64f51 --- /dev/null +++ b/include/clk/clk_domain.c @@ -0,0 +1,1666 @@ +/* + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTIO