aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRafael J. Wysocki <rafael.j.wysocki@intel.com>2017-04-28 17:14:00 -0400
committerRafael J. Wysocki <rafael.j.wysocki@intel.com>2017-04-28 17:14:00 -0400
commit0807ee0f525470482ecb002c45abcceed37e24e5 (patch)
tree171bac082f8053b6b6030d7669013049150a7816
parent5a7ad1146caa895ad718a534399e38bd2ba721b7 (diff)
parent2addac72afb304fa5aa57dbdd12659a8fd52d105 (diff)
Merge branch 'pm-cpufreq'
* pm-cpufreq: (37 commits) cpufreq: Add Tegra186 cpufreq driver cpufreq: imx6q: Fix error handling code cpufreq: imx6q: Set max suspend_freq to avoid changes during suspend cpufreq: imx6q: Fix handling EPROBE_DEFER from regulator cpufreq: schedutil: Use policy-dependent transition delays cpufreq: schedutil: Reduce frequencies slower cpufreq: intel_pstate: Add support for Gemini Lake cpufreq: intel_pstate: Eliminate intel_pstate_get_min_max() cpufreq: intel_pstate: Do not walk policy->cpus cpufreq: intel_pstate: Introduce pid_in_use() cpufreq: intel_pstate: Drop struct cpu_defaults cpufreq: intel_pstate: Move cpu_defaults definitions cpufreq: intel_pstate: Add update_util callback to pstate_funcs cpufreq: intel_pstate: Use different utilization update callbacks cpufreq: intel_pstate: Modify check in intel_pstate_update_status() cpufreq: intel_pstate: Drop driver_registered variable cpufreq: intel_pstate: Skip unnecessary PID resets on init cpufreq: intel_pstate: Set HWP sampling interval once cpufreq: intel_pstate: Clean up intel_pstate_busy_pid_reset() cpufreq: intel_pstate: Fold intel_pstate_reset_all_pid() into the caller ...
Diffstat
-rw-r--r--MAINTAINERS1
-rw-r--r--arch/arm/boot/dts/ste-dbx5x0.dtsi5
-rw-r--r--drivers/cpufreq/Kconfig.arm6
-rw-r--r--drivers/cpufreq/Makefile1
-rw-r--r--drivers/cpufreq/dbx500-cpufreq.c20
-rw-r--r--drivers/cpufreq/imx6q-cpufreq.c17
-rw-r--r--drivers/cpufreq/intel_pstate.c910
-rw-r--r--drivers/cpufreq/mt8173-cpufreq.c23
-rw-r--r--drivers/cpufreq/qoriq-cpufreq.c24
-rw-r--r--drivers/cpufreq/tegra186-cpufreq.c275
-rw-r--r--drivers/thermal/Kconfig12
-rw-r--r--drivers/thermal/Makefile1
-rw-r--r--drivers/thermal/db8500_cpufreq_cooling.c105
-rw-r--r--include/linux/cpufreq.h7
-rw-r--r--include/linux/tick.h1
-rw-r--r--kernel/sched/cpufreq_schedutil.c82
-rw-r--r--kernel/time/tick-sched.c12
17 files changed, 840 insertions, 662 deletions
diff --git a/MAINTAINERS b/MAINTAINERS
index 38d3e4ed7208..56a92ec7d3f7 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3463,6 +3463,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git
3463T: git git://git.linaro.org/people/vireshk/linux.git (For ARM Updates) 3463T: git git://git.linaro.org/people/vireshk/linux.git (For ARM Updates)
3464B: https://bugzilla.kernel.org 3464B: https://bugzilla.kernel.org
3465F: Documentation/cpu-freq/ 3465F: Documentation/cpu-freq/
3466F: Documentation/devicetree/bindings/cpufreq/
3466F: drivers/cpufreq/ 3467F: drivers/cpufreq/
3467F: include/linux/cpufreq.h 3468F: include/linux/cpufreq.h
3468F: tools/testing/selftests/cpufreq/ 3469F: tools/testing/selftests/cpufreq/
diff --git a/arch/arm/boot/dts/ste-dbx5x0.dtsi b/arch/arm/boot/dts/ste-dbx5x0.dtsi
index 162e1eb5373d..6c5affe2d0f5 100644
--- a/arch/arm/boot/dts/ste-dbx5x0.dtsi
+++ b/arch/arm/boot/dts/ste-dbx5x0.dtsi
@@ -1189,11 +1189,6 @@
1189 status = "disabled"; 1189 status = "disabled";
1190 }; 1190 };
1191 1191
1192 cpufreq-cooling {
1193 compatible = "stericsson,db8500-cpufreq-cooling";
1194 status = "disabled";
1195 };
1196
1197 mcde@a0350000 { 1192 mcde@a0350000 {
1198 compatible = "stericsson,mcde"; 1193 compatible = "stericsson,mcde";
1199 reg = <0xa0350000 0x1000>, /* MCDE */ 1194 reg = <0xa0350000 0x1000>, /* MCDE */
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index 74fa5c5904d3..74ed7e9a7f27 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -247,6 +247,12 @@ config ARM_TEGRA124_CPUFREQ
247 help 247 help
248 This adds the CPUFreq driver support for Tegra124 SOCs. 248 This adds the CPUFreq driver support for Tegra124 SOCs.
249 249
250config ARM_TEGRA186_CPUFREQ
251 tristate "Tegra186 CPUFreq support"
252 depends on ARCH_TEGRA && TEGRA_BPMP
253 help
254 This adds the CPUFreq driver support for Tegra186 SOCs.
255
250config ARM_TI_CPUFREQ 256config ARM_TI_CPUFREQ
251 bool "Texas Instruments CPUFreq support" 257 bool "Texas Instruments CPUFreq support"
252 depends on ARCH_OMAP2PLUS 258 depends on ARCH_OMAP2PLUS
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 9f5a8045f36d..b7e78f063c4f 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -77,6 +77,7 @@ obj-$(CONFIG_ARM_SPEAR_CPUFREQ) += spear-cpufreq.o
77obj-$(CONFIG_ARM_STI_CPUFREQ) += sti-cpufreq.o 77obj-$(CONFIG_ARM_STI_CPUFREQ) += sti-cpufreq.o
78obj-$(CONFIG_ARM_TEGRA20_CPUFREQ) += tegra20-cpufreq.o 78obj-$(CONFIG_ARM_TEGRA20_CPUFREQ) += tegra20-cpufreq.o
79obj-$(CONFIG_ARM_TEGRA124_CPUFREQ) += tegra124-cpufreq.o 79obj-$(CONFIG_ARM_TEGRA124_CPUFREQ) += tegra124-cpufreq.o
80obj-$(CONFIG_ARM_TEGRA186_CPUFREQ) += tegra186-cpufreq.o
80obj-$(CONFIG_ARM_TI_CPUFREQ) += ti-cpufreq.o 81obj-$(CONFIG_ARM_TI_CPUFREQ) += ti-cpufreq.o
81obj-$(CONFIG_ARM_VEXPRESS_SPC_CPUFREQ) += vexpress-spc-cpufreq.o 82obj-$(CONFIG_ARM_VEXPRESS_SPC_CPUFREQ) += vexpress-spc-cpufreq.o
82obj-$(CONFIG_ACPI_CPPC_CPUFREQ) += cppc_cpufreq.o 83obj-$(CONFIG_ACPI_CPPC_CPUFREQ) += cppc_cpufreq.o
diff --git a/drivers/cpufreq/dbx500-cpufreq.c b/drivers/cpufreq/dbx500-cpufreq.c
index 5c3ec1dd4921..3575b82210ba 100644
--- a/drivers/cpufreq/dbx500-cpufreq.c
+++ b/drivers/cpufreq/dbx500-cpufreq.c
@@ -11,6 +11,7 @@
11#include <linux/module.h> 11#include <linux/module.h>
12#include <linux/kernel.h> 12#include <linux/kernel.h>
13#include <linux/cpufreq.h> 13#include <linux/cpufreq.h>
14#include <linux/cpu_cooling.h>
14#include <linux/delay.h> 15#include <linux/delay.h>
15#include <linux/slab.h> 16#include <linux/slab.h>
16#include <linux/platform_device.h> 17#include <linux/platform_device.h>
@@ -18,6 +19,7 @@
18 19
19static struct cpufreq_frequency_table *freq_table; 20static struct cpufreq_frequency_table *freq_table;
20static struct clk *armss_clk; 21static struct clk *armss_clk;
22static struct thermal_cooling_device *cdev;
21 23
22static int dbx500_cpufreq_target(struct cpufreq_policy *policy, 24static int dbx500_cpufreq_target(struct cpufreq_policy *policy,
23 unsigned int index) 25 unsigned int index)
@@ -32,6 +34,22 @@ static int dbx500_cpufreq_init(struct cpufreq_policy *policy)
32 return cpufreq_generic_init(policy, freq_table, 20 * 1000); 34 return cpufreq_generic_init(policy, freq_table, 20 * 1000);
33} 35}
34 36
37static int dbx500_cpufreq_exit(struct cpufreq_policy *policy)
38{
39 if (!IS_ERR(cdev))
40 cpufreq_cooling_unregister(cdev);
41 return 0;
42}
43
44static void dbx500_cpufreq_ready(struct cpufreq_policy *policy)
45{
46 cdev = cpufreq_cooling_register(policy->cpus);
47 if (IS_ERR(cdev))
48 pr_err("Failed to register cooling device %ld\n", PTR_ERR(cdev));
49 else
50 pr_info("Cooling device registered: %s\n", cdev->type);
51}
52
35static struct cpufreq_driver dbx500_cpufreq_driver = { 53static struct cpufreq_driver dbx500_cpufreq_driver = {
36 .flags = CPUFREQ_STICKY | CPUFREQ_CONST_LOOPS | 54 .flags = CPUFREQ_STICKY | CPUFREQ_CONST_LOOPS |
37 CPUFREQ_NEED_INITIAL_FREQ_CHECK, 55 CPUFREQ_NEED_INITIAL_FREQ_CHECK,
@@ -39,6 +57,8 @@ static struct cpufreq_driver dbx500_cpufreq_driver = {
39 .target_index = dbx500_cpufreq_target, 57 .target_index = dbx500_cpufreq_target,
40 .get = cpufreq_generic_get, 58 .get = cpufreq_generic_get,
41 .init = dbx500_cpufreq_init, 59 .init = dbx500_cpufreq_init,
60 .exit = dbx500_cpufreq_exit,
61 .ready = dbx500_cpufreq_ready,
42 .name = "DBX500", 62 .name = "DBX500",
43 .attr = cpufreq_generic_attr, 63 .attr = cpufreq_generic_attr,
44}; 64};
diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c
index 7719b02e04f5..9c13f097fd8c 100644
--- a/drivers/cpufreq/imx6q-cpufreq.c
+++ b/drivers/cpufreq/imx6q-cpufreq.c
@@ -161,8 +161,13 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index)
161 161
162static int imx6q_cpufreq_init(struct cpufreq_policy *policy) 162static int imx6q_cpufreq_init(struct cpufreq_policy *policy)
163{ 163{
164 int ret;
165
164 policy->clk = arm_clk; 166 policy->clk = arm_clk;
165 return cpufreq_generic_init(policy, freq_table, transition_latency); 167 ret = cpufreq_generic_init(policy, freq_table, transition_latency);
168 policy->suspend_freq = policy->max;
169
170 return ret;
166} 171}
167 172
168static struct cpufreq_driver imx6q_cpufreq_driver = { 173static struct cpufreq_driver imx6q_cpufreq_driver = {
@@ -173,6 +178,7 @@ static struct cpufreq_driver imx6q_cpufreq_driver = {
173 .init = imx6q_cpufreq_init, 178 .init = imx6q_cpufreq_init,
174 .name = "imx6q-cpufreq", 179 .name = "imx6q-cpufreq",
175 .attr = cpufreq_generic_attr, 180 .attr = cpufreq_generic_attr,
181 .suspend = cpufreq_generic_suspend,
176}; 182};
177 183
178static int imx6q_cpufreq_probe(struct platform_device *pdev) 184static int imx6q_cpufreq_probe(struct platform_device *pdev)
@@ -222,6 +228,13 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev)
222 arm_reg = regulator_get(cpu_dev, "arm"); 228 arm_reg = regulator_get(cpu_dev, "arm");
223 pu_reg = regulator_get_optional(cpu_dev, "pu"); 229 pu_reg = regulator_get_optional(cpu_dev, "pu");
224 soc_reg = regulator_get(cpu_dev, "soc"); 230 soc_reg = regulator_get(cpu_dev, "soc");
231 if (PTR_ERR(arm_reg) == -EPROBE_DEFER ||
232 PTR_ERR(soc_reg) == -EPROBE_DEFER ||
233 PTR_ERR(pu_reg) == -EPROBE_DEFER) {
234 ret = -EPROBE_DEFER;
235 dev_dbg(cpu_dev, "regulators not ready, defer\n");
236 goto put_reg;
237 }
225 if (IS_ERR(arm_reg) || IS_ERR(soc_reg)) { 238 if (IS_ERR(arm_reg) || IS_ERR(soc_reg)) {
226 dev_err(cpu_dev, "failed to get regulators\n"); 239 dev_err(cpu_dev, "failed to get regulators\n");
227 ret = -ENOENT; 240 ret = -ENOENT;
@@ -255,7 +268,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev)
255 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); 268 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
256 if (ret) { 269 if (ret) {
257 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); 270 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
258 goto put_reg; 271 goto out_free_opp;
259 } 272 }
260 273
261 /* Make imx6_soc_volt array's size same as arm opp number */ 274 /* Make imx6_soc_volt array's size same as arm opp number */
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 283491f742d3..b7de5bd76a31 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -37,7 +37,11 @@
37#include <asm/cpufeature.h> 37#include <asm/cpufeature.h>
38#include <asm/intel-family.h> 38#include <asm/intel-family.h>
39 39
40#define INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
41#define INTEL_PSTATE_HWP_SAMPLING_INTERVAL (50 * NSEC_PER_MSEC)
42
40#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 43#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
44#define INTEL_CPUFREQ_TRANSITION_DELAY 500
41 45
42#ifdef CONFIG_ACPI 46#ifdef CONFIG_ACPI
43#include <acpi/processor.h> 47#include <acpi/processor.h>
@@ -74,6 +78,11 @@ static inline int ceiling_fp(int32_t x)
74 return ret; 78 return ret;
75} 79}
76 80
81static inline int32_t percent_fp(int percent)
82{
83 return div_fp(percent, 100);
84}
85
77static inline u64 mul_ext_fp(u64 x, u64 y) 86static inline u64 mul_ext_fp(u64 x, u64 y)
78{ 87{
79 return (x * y) >> EXT_FRAC_BITS; 88 return (x * y) >> EXT_FRAC_BITS;
@@ -186,45 +195,22 @@ struct _pid {
186}; 195};
187 196
188/** 197/**
189 * struct perf_limits - Store user and policy limits 198 * struct global_params - Global parameters, mostly tunable via sysfs.
190 * @no_turbo: User requested turbo state from intel_pstate sysfs 199 * @no_turbo: Whether or not to use turbo P-states.
191 * @turbo_disabled: Platform turbo status either from msr 200 * @turbo_disabled: Whethet or not turbo P-states are available at all,
192 * MSR_IA32_MISC_ENABLE or when maximum available pstate 201 * based on the MSR_IA32_MISC_ENABLE value and whether or
193 * matches the maximum turbo pstate 202 * not the maximum reported turbo P-state is different from
194 * @max_perf_pct: Effective maximum performance limit in percentage, this 203 * the maximum reported non-turbo one.
195 * is minimum of either limits enforced by cpufreq policy 204 * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
196 * or limits from user set limits via intel_pstate sysfs 205 * P-state capacity.
197 * @min_perf_pct: Effective minimum performance limit in percentage, this 206 * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
198 * is maximum of either limits enforced by cpufreq policy 207 * P-state capacity.
199 * or limits from user set limits via intel_pstate sysfs
200 * @max_perf: This is a scaled value between 0 to 255 for max_perf_pct
201 * This value is used to limit max pstate
202 * @min_perf: This is a scaled value between 0 to 255 for min_perf_pct
203 * This value is used to limit min pstate
204 * @max_policy_pct: The maximum performance in percentage enforced by
205 * cpufreq setpolicy interface
206 * @max_sysfs_pct: The maximum performance in percentage enforced by
207 * intel pstate sysfs interface, unused when per cpu
208 * controls are enforced
209 * @min_policy_pct: The minimum performance in percentage enforced by
210 * cpufreq setpolicy interface
211 * @min_sysfs_pct: The minimum performance in percentage enforced by
212 * intel pstate sysfs interface, unused when per cpu
213 * controls are enforced
214 *
215 * Storage for user and policy defined limits.
216 */ 208 */
217struct perf_limits { 209struct global_params {
218 int no_turbo; 210 bool no_turbo;
219 int turbo_disabled; 211 bool turbo_disabled;
220 int max_perf_pct; 212 int max_perf_pct;
221 int min_perf_pct; 213 int min_perf_pct;
222 int32_t max_perf;
223 int32_t min_perf;
224 int max_policy_pct;
225 int max_sysfs_pct;
226 int min_policy_pct;
227 int min_sysfs_pct;
228}; 214};
229 215
230/** 216/**
@@ -245,9 +231,10 @@ struct perf_limits {
245 * @prev_cummulative_iowait: IO Wait time difference from last and 231 * @prev_cummulative_iowait: IO Wait time difference from last and
246 * current sample 232 * current sample
247 * @sample: Storage for storing last Sample data 233 * @sample: Storage for storing last Sample data
248 * @perf_limits: Pointer to perf_limit unique to this CPU 234 * @min_perf: Minimum capacity limit as a fraction of the maximum
249 * Not all field in the structure are applicable 235 * turbo P-state capacity.
250 * when per cpu controls are enforced 236 * @max_perf: Maximum capacity limit as a fraction of the maximum
237 * turbo P-state capacity.
251 * @acpi_perf_data: Stores ACPI perf information read from _PSS 238 * @acpi_perf_data: Stores ACPI perf information read from _PSS
252 * @valid_pss_table: Set to true for valid ACPI _PSS entries found 239 * @valid_pss_table: Set to true for valid ACPI _PSS entries found
253 * @epp_powersave: Last saved HWP energy performance preference 240 * @epp_powersave: Last saved HWP energy performance preference
@@ -279,7 +266,8 @@ struct cpudata {
279 u64 prev_tsc; 266 u64 prev_tsc;
280 u64 prev_cummulative_iowait; 267 u64 prev_cummulative_iowait;
281 struct sample sample; 268 struct sample sample;
282 struct perf_limits *perf_limits; 269 int32_t min_perf;
270 int32_t max_perf;
283#ifdef CONFIG_ACPI 271#ifdef CONFIG_ACPI
284 struct acpi_processor_performance acpi_perf_data; 272 struct acpi_processor_performance acpi_perf_data;
285 bool valid_pss_table; 273 bool valid_pss_table;
@@ -324,7 +312,7 @@ struct pstate_adjust_policy {
324 * @get_scaling: Callback to get frequency scaling factor 312 * @get_scaling: Callback to get frequency scaling factor
325 * @get_val: Callback to convert P state to actual MSR write value 313 * @get_val: Callback to convert P state to actual MSR write value
326 * @get_vid: Callback to get VID data for Atom platforms 314 * @get_vid: Callback to get VID data for Atom platforms
327 * @get_target_pstate: Callback to a function to calculate next P state to use 315 * @update_util: Active mode utilization update callback.
328 * 316 *
329 * Core and Atom CPU models have different way to get P State limits. This 317 * Core and Atom CPU models have different way to get P State limits. This
330 * structure is used to store those callbacks. 318 * structure is used to store those callbacks.
@@ -337,43 +325,31 @@ struct pstate_funcs {
337 int (*get_scaling)(void); 325 int (*get_scaling)(void);
338 u64 (*get_val)(struct cpudata*, int pstate); 326 u64 (*get_val)(struct cpudata*, int pstate);
339 void (*get_vid)(struct cpudata *); 327 void (*get_vid)(struct cpudata *);
340 int32_t (*get_target_pstate)(struct cpudata *); 328 void (*update_util)(struct update_util_data *data, u64 time,
329 unsigned int flags);
341}; 330};
342 331
343/** 332static struct pstate_funcs pstate_funcs __read_mostly;
344 * struct cpu_defaults- Per CPU model default config data 333static struct pstate_adjust_policy pid_params __read_mostly = {
345 * @pid_policy: PID config data 334 .sample_rate_ms = 10,
346 * @funcs: Callback function data 335 .sample_rate_ns = 10 * NSEC_PER_MSEC,
347 */ 336 .deadband = 0,
348struct cpu_defaults { 337 .setpoint = 97,
349 struct pstate_adjust_policy pid_policy; 338 .p_gain_pct = 20,
350 struct pstate_funcs funcs; 339 .d_gain_pct = 0,
340 .i_gain_pct = 0,
351}; 341};
352 342
353static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu);
354static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu);
355
356static struct pstate_adjust_policy pid_params __read_mostly;
357static struct pstate_funcs pstate_funcs __read_mostly;
358static int hwp_active __read_mostly; 343static int hwp_active __read_mostly;
359static bool per_cpu_limits __read_mostly; 344static bool per_cpu_limits __read_mostly;
360 345
361static bool driver_registered __read_mostly; 346static struct cpufreq_driver *intel_pstate_driver __read_mostly;
362 347
363#ifdef CONFIG_ACPI 348#ifdef CONFIG_ACPI
364static bool acpi_ppc; 349static bool acpi_ppc;
365#endif 350#endif
366 351
367static struct perf_limits global; 352static struct global_params global;
368
369static void intel_pstate_init_limits(struct perf_limits *limits)
370{
371 memset(limits, 0, sizeof(*limits));
372 limits->max_perf_pct = 100;
373 limits->max_perf = int_ext_tofp(1);
374 limits->max_policy_pct = 100;
375 limits->max_sysfs_pct = 100;
376}
377 353
378static DEFINE_MUTEX(intel_pstate_driver_lock); 354static DEFINE_MUTEX(intel_pstate_driver_lock);
379static DEFINE_MUTEX(intel_pstate_limits_lock); 355static DEFINE_MUTEX(intel_pstate_limits_lock);
@@ -530,29 +506,6 @@ static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
530} 506}
531#endif 507#endif
532 508
533static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
534 int deadband, int integral) {
535 pid->setpoint = int_tofp(setpoint);
536 pid->deadband = int_tofp(deadband);
537 pid->integral = int_tofp(integral);
538 pid->last_err = int_tofp(setpoint) - int_tofp(busy);
539}
540
541static inline void pid_p_gain_set(struct _pid *pid, int percent)
542{
543 pid->p_gain = div_fp(percent, 100);
544}
545
546static inline void pid_i_gain_set(struct _pid *pid, int percent)
547{
548 pid->i_gain = div_fp(percent, 100);
549}
550
551static inline void pid_d_gain_set(struct _pid *pid, int percent)
552{
553 pid->d_gain = div_fp(percent, 100);
554}
555
556static signed int pid_calc(struct _pid *pid, int32_t busy) 509static signed int pid_calc(struct _pid *pid, int32_t busy)
557{ 510{
558 signed int result; 511 signed int result;
@@ -590,23 +543,17 @@ static signed int pid_calc(struct _pid *pid, int32_t busy)
590 return (signed int)fp_toint(result); 543 return (signed int)fp_toint(result);
591} 544}
592 545
593static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu) 546static inline void intel_pstate_pid_reset(struct cpudata *cpu)
594{
595 pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
596 pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
597 pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
598
599 pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0);
600}
601
602static inline void intel_pstate_reset_all_pid(void)
603{ 547{
604 unsigned int cpu; 548 struct _pid *pid = &cpu->pid;
605 549
606 for_each_online_cpu(cpu) { 550 pid->p_gain = percent_fp(pid_params.p_gain_pct);
607 if (all_cpu_data[cpu]) 551 pid->d_gain = percent_fp(pid_params.d_gain_pct);
608 intel_pstate_busy_pid_reset(all_cpu_data[cpu]); 552 pid->i_gain = percent_fp(pid_params.i_gain_pct);
609 } 553 pid->setpoint = int_tofp(pid_params.setpoint);
554 pid->last_err = pid->setpoint - int_tofp(100);
555 pid->deadband = int_tofp(pid_params.deadband);
556 pid->integral = 0;
610} 557}
611 558
612static inline void update_turbo_state(void) 559static inline void update_turbo_state(void)
@@ -621,6 +568,14 @@ static inline void update_turbo_state(void)
621 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate); 568 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
622} 569}
623 570
571static int min_perf_pct_min(void)
572{
573 struct cpudata *cpu = all_cpu_data[0];
574
575 return DIV_ROUND_UP(cpu->pstate.min_pstate * 100,
576 cpu->pstate.turbo_pstate);
577}
578
624static s16 intel_pstate_get_epb(struct cpudata *cpu_data) 579static s16 intel_pstate_get_epb(struct cpudata *cpu_data)
625{ 580{
626 u64 epb; 581 u64 epb;
@@ -838,96 +793,80 @@ static struct freq_attr *hwp_cpufreq_attrs[] = {
838 NULL, 793 NULL,
839}; 794};
840 795
841static void intel_pstate_hwp_set(struct cpufreq_policy *policy) 796static void intel_pstate_hwp_set(unsigned int cpu)
842{ 797{
843 int min, hw_min, max, hw_max, cpu; 798 struct cpudata *cpu_data = all_cpu_data[cpu];
844 struct perf_limits *perf_limits = &global; 799 int min, hw_min, max, hw_max;
845 u64 value, cap; 800 u64 value, cap;
801 s16 epp;
846 802
847 for_each_cpu(cpu, policy->cpus) { 803 rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
848 struct cpudata *cpu_data = all_cpu_data[cpu]; 804 hw_min = HWP_LOWEST_PERF(cap);
849 s16 epp; 805 if (global.no_turbo)
850 806 hw_max = HWP_GUARANTEED_PERF(cap);
851 if (per_cpu_limits) 807 else
852 perf_limits = all_cpu_data[cpu]->perf_limits; 808 hw_max = HWP_HIGHEST_PERF(cap);
853
854 rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
855 hw_min = HWP_LOWEST_PERF(cap);
856 if (global.no_turbo)
857 hw_max = HWP_GUARANTEED_PERF(cap);
858 else
859 hw_max = HWP_HIGHEST_PERF(cap);
860
861 max = fp_ext_toint(hw_max * perf_limits->max_perf);
862 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
863 min = max;
864 else
865 min = fp_ext_toint(hw_max * perf_limits->min_perf);
866 809
867 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); 810 max = fp_ext_toint(hw_max * cpu_data->max_perf);
811 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
812 min = max;
813 else
814 min = fp_ext_toint(hw_max * cpu_data->min_perf);
868 815
869 value &= ~HWP_MIN_PERF(~0L); 816 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
870 value |= HWP_MIN_PERF(min);
871 817
872 value &= ~HWP_MAX_PERF(~0L); 818 value &= ~HWP_MIN_PERF(~0L);
873 value |= HWP_MAX_PERF(max); 819 value |= HWP_MIN_PERF(min);
874 820
875 if (cpu_data->epp_policy == cpu_data->policy) 821 value &= ~HWP_MAX_PERF(~0L);
876 goto skip_epp; 822 value |= HWP_MAX_PERF(max);
877 823
878 cpu_data->epp_policy = cpu_data->policy; 824 if (cpu_data->epp_policy == cpu_data->policy)
825 goto skip_epp;
879 826
880 if (cpu_data->epp_saved >= 0) { 827 cpu_data->epp_policy = cpu_data->policy;
881 epp = cpu_data->epp_saved;
882 cpu_data->epp_saved = -EINVAL;
883 goto update_epp;
884 }
885 828
886 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) { 829 if (cpu_data->epp_saved >= 0) {
887 epp = intel_pstate_get_epp(cpu_data, value); 830 epp = cpu_data->epp_saved;
888 cpu_data->epp_powersave = epp; 831 cpu_data->epp_saved = -EINVAL;
889 /* If EPP read was failed, then don't try to write */ 832 goto update_epp;
890 if (epp < 0) 833 }
891 goto skip_epp;
892 834
835 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
836 epp = intel_pstate_get_epp(cpu_data, value);
837 cpu_data->epp_powersave = epp;
838 /* If EPP read was failed, then don't try to write */
839 if (epp < 0)
840 goto skip_epp;
893 841
894 epp = 0; 842 epp = 0;
895 } else { 843 } else {
896 /* skip setting EPP, when saved value is invalid */ 844 /* skip setting EPP, when saved value is invalid */
897 if (cpu_data->epp_powersave < 0) 845 if (cpu_data->epp_powersave < 0)
898 goto skip_epp; 846 goto skip_epp;
899 847
900 /* 848 /*
901 * No need to restore EPP when it is not zero. This 849 * No need to restore EPP when it is not zero. This
902 * means: 850 * means:
903 * - Policy is not changed 851 * - Policy is not changed
904 * - user has manually changed 852 * - user has manually changed
905 * - Error reading EPB 853 * - Error reading EPB
906 */ 854 */
907 epp = intel_pstate_get_epp(cpu_data, value); 855 epp = intel_pstate_get_epp(cpu_data, value);
908 if (epp) 856 if (epp)
909 goto skip_epp; 857 goto skip_epp;
910 858
911 epp = cpu_data->epp_powersave; 859 epp = cpu_data->epp_powersave;
912 } 860 }
913update_epp: 861update_epp:
914 if (static_cpu_has(X86_FEATURE_HWP_EPP)) { 862 if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
915 value &= ~GENMASK_ULL(31, 24); 863 value &= ~GENMASK_ULL(31, 24);
916 value |= (u64)epp << 24; 864 value |= (u64)epp << 24;
917 } else { 865 } else {
918 intel_pstate_set_epb(cpu, epp); 866 intel_pstate_set_epb(cpu, epp);
919 }
920skip_epp:
921 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
922 } 867 }
923} 868skip_epp:
924 869 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
925static int intel_pstate_hwp_set_policy(struct cpufreq_policy *policy)
926{
927 if (hwp_active)
928 intel_pstate_hwp_set(policy);
929
930 return 0;
931} 870}
932 871
933static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy) 872static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
@@ -944,20 +883,17 @@ static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
944 883
945static int intel_pstate_resume(struct cpufreq_policy *policy) 884static int intel_pstate_resume(struct cpufreq_policy *policy)
946{ 885{
947 int ret;
948
949 if (!hwp_active) 886 if (!hwp_active)
950 return 0; 887 return 0;
951 888
952 mutex_lock(&intel_pstate_limits_lock); 889 mutex_lock(&intel_pstate_limits_lock);
953 890
954 all_cpu_data[policy->cpu]->epp_policy = 0; 891 all_cpu_data[policy->cpu]->epp_policy = 0;
955 892 intel_pstate_hwp_set(policy->cpu);
956 ret = intel_pstate_hwp_set_policy(policy);
957 893
958 mutex_unlock(&intel_pstate_limits_lock); 894 mutex_unlock(&intel_pstate_limits_lock);
959 895
960 return ret; 896 return 0;
961} 897}
962 898
963static void intel_pstate_update_policies(void) 899static void intel_pstate_update_policies(void)
@@ -971,9 +907,14 @@ static void intel_pstate_update_policies(void)
971/************************** debugfs begin ************************/ 907/************************** debugfs begin ************************/
972static int pid_param_set(void *data, u64 val) 908static int pid_param_set(void *data, u64 val)
973{ 909{
910 unsigned int cpu;
911
974 *(u32 *)data = val; 912 *(u32 *)data = val;
975 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; 913 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
976 intel_pstate_reset_all_pid(); 914 for_each_possible_cpu(cpu)
915 if (all_cpu_data[cpu])
916 intel_pstate_pid_reset(all_cpu_data[cpu]);
917
977 return 0; 918 return 0;
978} 919}
979 920
@@ -1084,7 +1025,7 @@ static ssize_t show_turbo_pct(struct kobject *kobj,
1084 1025
1085 mutex_lock(&intel_pstate_driver_lock); 1026 mutex_lock(&intel_pstate_driver_lock);
1086 1027
1087 if (!driver_registered) { 1028 if (!intel_pstate_driver) {
1088 mutex_unlock(&intel_pstate_driver_lock); 1029 mutex_unlock(&intel_pstate_driver_lock);
1089 return -EAGAIN; 1030 return -EAGAIN;
1090 } 1031 }
@@ -1109,7 +1050,7 @@ static ssize_t show_num_pstates(struct kobject *kobj,
1109 1050
1110 mutex_lock(&intel_pstate_driver_lock); 1051 mutex_lock(&intel_pstate_driver_lock);
1111 1052
1112 if (!driver_registered) { 1053 if (!intel_pstate_driver) {
1113 mutex_unlock(&intel_pstate_driver_lock); 1054 mutex_unlock(&intel_pstate_driver_lock);
1114 return -EAGAIN; 1055 return -EAGAIN;
1115 } 1056 }
@@ -1129,7 +1070,7 @@ static ssize_t show_no_turbo(struct kobject *kobj,
1129 1070
1130 mutex_lock(&intel_pstate_driver_lock); 1071 mutex_lock(&intel_pstate_driver_lock);
1131 1072
1132 if (!driver_registered) { 1073 if (!intel_pstate_driver) {
1133 mutex_unlock(&intel_pstate_driver_lock); 1074 mutex_unlock(&intel_pstate_driver_lock);
1134 return -EAGAIN; 1075 return -EAGAIN;
1135 } 1076 }
@@ -1157,7 +1098,7 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
1157 1098
1158 mutex_lock(&intel_pstate_driver_lock); 1099 mutex_lock(&intel_pstate_driver_lock);
1159 1100
1160 if (!driver_registered) { 1101 if (!intel_pstate_driver) {
1161 mutex_unlock(&intel_pstate_driver_lock); 1102 mutex_unlock(&intel_pstate_driver_lock);
1162 return -EAGAIN; 1103 return -EAGAIN;
1163 } 1104 }
@@ -1174,6 +1115,15 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
1174 1115
1175 global.no_turbo = clamp_t(int, input, 0, 1); 1116 global.no_turbo = clamp_t(int, input, 0, 1);
1176 1117
1118 if (global.no_turbo) {
1119 struct cpudata *cpu = all_cpu_data[0];
1120 int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate;
1121
1122 /* Squash the global minimum into the permitted range. */
1123 if (global.min_perf_pct > pct)
1124 global.min_perf_pct = pct;
1125 }
1126
1177 mutex_unlock(&intel_pstate_limits_lock); 1127 mutex_unlock(&intel_pstate_limits_lock);
1178 1128
1179 intel_pstate_update_policies(); 1129 intel_pstate_update_policies();
@@ -1195,18 +1145,14 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
1195 1145
1196 mutex_lock(&intel_pstate_driver_lock); 1146 mutex_lock(&intel_pstate_driver_lock);
1197 1147
1198 if (!driver_registered) { 1148 if (!intel_pstate_driver) {
1199 mutex_unlock(&intel_pstate_driver_lock); 1149 mutex_unlock(&intel_pstate_driver_lock);
1200 return -EAGAIN; 1150 return -EAGAIN;
1201 } 1151 }
1202 1152
1203 mutex_lock(&intel_pstate_limits_lock); 1153 mutex_lock(&intel_pstate_limits_lock);
1204 1154
1205 global.max_sysfs_pct = clamp_t(int, input, 0 , 100); 1155 global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100);
1206 global.max_perf_pct = min(global.max_policy_pct, global.max_sysfs_pct);
1207 global.max_perf_pct = max(global.min_policy_pct, global.max_perf_pct);
1208 global.max_perf_pct = max(global.min_perf_pct, global.max_perf_pct);
1209 global.max_perf = percent_ext_fp(global.max_perf_pct);
1210 1156
1211 mutex_unlock(&intel_pstate_limits_lock); 1157 mutex_unlock(&intel_pstate_limits_lock);
1212 1158
@@ -1229,18 +1175,15 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
1229 1175
1230 mutex_lock(&intel_pstate_driver_lock); 1176 mutex_lock(&intel_pstate_driver_lock);
1231 1177
1232 if (!driver_registered) { 1178 if (!intel_pstate_driver) {
1233 mutex_unlock(&intel_pstate_driver_lock); 1179 mutex_unlock(&intel_pstate_driver_lock);
1234 return -EAGAIN; 1180 return -EAGAIN;
1235 } 1181 }
1236 1182
1237 mutex_lock(&intel_pstate_limits_lock); 1183 mutex_lock(&intel_pstate_limits_lock);
1238 1184
1239 global.min_sysfs_pct = clamp_t(int, input, 0 , 100); 1185 global.min_perf_pct = clamp_t(int, input,
1240 global.min_perf_pct = max(global.min_policy_pct, global.min_sysfs_pct); 1186 min_perf_pct_min(), global.max_perf_pct);
1241 global.min_perf_pct = min(global.max_policy_pct, global.min_perf_pct);
1242 global.min_perf_pct = min(global.max_perf_pct, global.min_perf_pct);
1243 global.min_perf = percent_ext_fp(global.min_perf_pct);
1244 1187
1245 mutex_unlock(&intel_pstate_limits_lock); 1188 mutex_unlock(&intel_pstate_limits_lock);
1246 1189
@@ -1554,132 +1497,10 @@ static int knl_get_turbo_pstate(void)
1554 return ret; 1497 return ret;
1555} 1498}
1556 1499
1557static struct cpu_defaults core_params = { 1500static int intel_pstate_get_base_pstate(struct cpudata *cpu)
1558 .pid_policy = {
1559 .sample_rate_ms = 10,
1560 .deadband = 0,
1561 .setpoint = 97,
1562 .p_gain_pct = 20,
1563 .d_gain_pct = 0,
1564 .i_gain_pct = 0,
1565 },
1566 .funcs = {
1567 .get_max = core_get_max_pstate,
1568 .get_max_physical = core_get_max_pstate_physical,
1569 .get_min = core_get_min_pstate,
1570 .get_turbo = core_get_turbo_pstate,
1571 .get_scaling = core_get_scaling,
1572 .get_val = core_get_val,
1573 .get_target_pstate = get_target_pstate_use_performance,
1574 },
1575};
1576
1577static const struct cpu_defaults silvermont_params = {
1578 .pid_policy = {
1579 .sample_rate_ms = 10,
1580 .deadband = 0,
1581 .setpoint = 60,
1582 .p_gain_pct = 14,
1583 .d_gain_pct = 0,
1584 .i_gain_pct = 4,
1585 },
1586 .funcs = {
1587 .get_max = atom_get_max_pstate,
1588 .get_max_physical = atom_get_max_pstate,
1589 .get_min = atom_get_min_pstate,
1590 .get_turbo = atom_get_turbo_pstate,
1591 .get_val = atom_get_val,
1592 .get_scaling = silvermont_get_scaling,
1593 .get_vid = atom_get_vid,
1594 .get_target_pstate = get_target_pstate_use_cpu_load,
1595 },
1596};
1597
1598static const struct cpu_defaults airmont_params = {
1599 .pid_policy = {
1600 .sample_rate_ms = 10,
1601 .deadband = 0,
1602 .setpoint = 60,
1603 .p_gain_pct = 14,
1604 .d_gain_pct = 0,
1605 .i_gain_pct = 4,
1606 },
1607 .funcs = {
1608 .get_max = atom_get_max_pstate,
1609 .get_max_physical = atom_get_max_pstate,
1610 .get_min = atom_get_min_pstate,
1611 .get_turbo = atom_get_turbo_pstate,
1612 .get_val = atom_get_val,
1613 .get_scaling = airmont_get_scaling,
1614 .get_vid = atom_get_vid,
1615 .get_target_pstate = get_target_pstate_use_cpu_load,
1616 },
1617};
1618
1619static const struct cpu_defaults knl_params = {
1620 .pid_policy = {
1621 .sample_rate_ms = 10,
1622 .deadband = 0,
1623 .setpoint = 97,
1624 .p_gain_pct = 20,
1625 .d_gain_pct = 0,
1626 .i_gain_pct = 0,
1627 },
1628 .funcs = {
1629 .get_max = core_get_max_pstate,
1630 .get_max_physical = core_get_max_pstate_physical,
1631 .get_min = core_get_min_pstate,
1632 .get_turbo = knl_get_turbo_pstate,
1633 .get_scaling = core_get_scaling,
1634 .get_val = core_get_val,
1635 .get_target_pstate = get_target_pstate_use_performance,
1636 },
1637};
1638
1639static const struct cpu_defaults bxt_params = {
1640 .pid_policy = {
1641 .sample_rate_ms = 10,
1642 .deadband = 0,
1643 .setpoint = 60,
1644 .p_gain_pct = 14,
1645 .d_gain_pct = 0,
1646 .i_gain_pct = 4,
1647 },
1648 .funcs = {
1649 .get_max = core_get_max_pstate,
1650 .get_max_physical = core_get_max_pstate_physical,
1651 .get_min = core_get_min_pstate,
1652 .get_turbo = core_get_turbo_pstate,
1653 .get_scaling = core_get_scaling,
1654 .get_val = core_get_val,
1655 .get_target_pstate = get_target_pstate_use_cpu_load,
1656 },
1657};
1658
1659static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
1660{ 1501{
1661 int max_perf = cpu->pstate.turbo_pstate; 1502 return global.no_turbo || global.turbo_disabled ?
1662 int max_perf_adj; 1503 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1663 int min_perf;
1664 struct perf_limits *perf_limits = &global;
1665
1666 if (global.no_turbo || global.turbo_disabled)
1667 max_perf = cpu->pstate.max_pstate;
1668
1669 if (per_cpu_limits)
1670 perf_limits = cpu->perf_limits;
1671
1672 /*
1673 * performance can be limited by user through sysfs, by cpufreq
1674 * policy, or by cpu specific default values determined through
1675 * experimentation.
1676 */
1677 max_perf_adj = fp_ext_toint(max_perf * perf_limits->max_perf);
1678 *max = clamp_t(int, max_perf_adj,
1679 cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
1680
1681 min_perf = fp_ext_toint(max_perf * perf_limits->min_perf);
1682 *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
1683} 1504}
1684 1505
1685static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) 1506static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
@@ -1702,11 +1523,13 @@ static void intel_pstate_set_min_pstate(struct cpudata *cpu)
1702 1523
1703static void intel_pstate_max_within_limits(struct cpudata *cpu) 1524static void intel_pstate_max_within_limits(struct cpudata *cpu)
1704{ 1525{
1705 int min_pstate, max_pstate; 1526 int pstate;
1706 1527
1707 update_turbo_state(); 1528 update_turbo_state();
1708 intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate); 1529 pstate = intel_pstate_get_base_pstate(cpu);
1709 intel_pstate_set_pstate(cpu, max_pstate); 1530 pstate = max(cpu->pstate.min_pstate,
1531 fp_ext_toint(pstate * cpu->max_perf));
1532 intel_pstate_set_pstate(cpu, pstate);
1710} 1533}
1711 1534
1712static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) 1535static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
@@ -1767,7 +1590,11 @@ static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
1767 * that sample.time will always be reset before setting the utilization 1590 * that sample.time will always be reset before setting the utilization
1768 * update hook and make the caller skip the sample then. 1591 * update hook and make the caller skip the sample then.
1769 */ 1592 */
1770 return !!cpu->last_sample_time; 1593 if (cpu->last_sample_time) {
1594 intel_pstate_calc_avg_perf(cpu);
1595 return true;
1596 }
1597 return false;
1771} 1598}
1772 1599
1773static inline int32_t get_avg_frequency(struct cpudata *cpu) 1600static inline int32_t get_avg_frequency(struct cpudata *cpu)
@@ -1788,6 +1615,9 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
1788 int32_t busy_frac, boost; 1615 int32_t busy_frac, boost;
1789 int target, avg_pstate; 1616 int target, avg_pstate;
1790 1617
1618 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE)
1619 return cpu->pstate.turbo_pstate;
1620
1791 busy_frac = div_fp(sample->mperf, sample->tsc); 1621 busy_frac = div_fp(sample->mperf, sample->tsc);
1792 1622
1793 boost = cpu->iowait_boost; 1623 boost = cpu->iowait_boost;
@@ -1824,6 +1654,9 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
1824 int32_t perf_scaled, max_pstate, current_pstate, sample_ratio; 1654 int32_t perf_scaled, max_pstate, current_pstate, sample_ratio;
1825 u64 duration_ns; 1655 u64 duration_ns;
1826 1656
1657 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE)
1658 return cpu->pstate.turbo_pstate;
1659
1827 /* 1660 /*
1828 * perf_scaled is the ratio of the average P-state during the last 1661 * perf_scaled is the ratio of the average P-state during the last
1829 * sampling period to the P-state requested last time (in percent). 1662 * sampling period to the P-state requested last time (in percent).
@@ -1858,11 +1691,13 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
1858 1691
1859static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate) 1692static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
1860{ 1693{
1861 int max_perf, min_perf; 1694 int max_pstate = intel_pstate_get_base_pstate(cpu);
1695 int min_pstate;
1862 1696
1863 intel_pstate_get_min_max(cpu, &min_perf, &max_perf); 1697 min_pstate = max(cpu->pstate.min_pstate,
1864 pstate = clamp_t(int, pstate, min_perf, max_perf); 1698 fp_ext_toint(max_pstate * cpu->min_perf));
1865 return pstate; 1699 max_pstate = max(min_pstate, fp_ext_toint(max_pstate * cpu->max_perf));
1700 return clamp_t(int, pstate, min_pstate, max_pstate);
1866} 1701}
1867 1702
1868static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) 1703static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
@@ -1874,16 +1709,11 @@ static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
1874 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); 1709 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
1875} 1710}
1876 1711
1877static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) 1712static void intel_pstate_adjust_pstate(struct cpudata *cpu, int target_pstate)
1878{ 1713{
1879 int from, target_pstate; 1714 int from = cpu->pstate.current_pstate;
1880 struct sample *sample; 1715 struct sample *sample;
1881 1716
1882 from = cpu->pstate.current_pstate;
1883
1884 target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ?
1885 cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu);
1886
1887 update_turbo_state(); 1717 update_turbo_state();
1888 1718
1889 target_pstate = intel_pstate_prepare_request(cpu, target_pstate); 1719 target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
@@ -1902,76 +1732,155 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
1902 fp_toint(cpu->iowait_boost * 100)); 1732 fp_toint(cpu->iowait_boost * 100));
1903} 1733}
1904 1734
1735static void intel_pstate_update_util_hwp(struct update_util_data *data,
1736 u64 time, unsigned int flags)
1737{
1738 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1739 u64 delta_ns = time - cpu->sample.time;
1740
1741 if ((s64)delta_ns >= INTEL_PSTATE_HWP_SAMPLING_INTERVAL)
1742 intel_pstate_sample(cpu, time);
1743}
1744
1745static void intel_pstate_update_util_pid(struct update_util_data *data,
1746 u64 time, unsigned int flags)
1747{
1748 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1749 u64 delta_ns = time - cpu->sample.time;
1750
1751 if ((s64)delta_ns < pid_params.sample_rate_ns)
1752 return;
1753
1754 if (intel_pstate_sample(cpu, time)) {
1755 int target_pstate;
1756
1757 target_pstate = get_target_pstate_use_performance(cpu);
1758 intel_pstate_adjust_pstate(cpu, target_pstate);
1759 }
1760}
1761
1905static void intel_pstate_update_util(struct update_util_data *data, u64 time, 1762static void intel_pstate_update_util(struct update_util_data *data, u64 time,
1906 unsigned int flags) 1763 unsigned int flags)
1907{ 1764{
1908 struct cpudata *cpu = container_of(data, struct cpudata, update_util); 1765 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1909 u64 delta_ns; 1766 u64 delta_ns;
1910 1767
1911 if (pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load) { 1768 if (flags & SCHED_CPUFREQ_IOWAIT) {
1912 if (flags & SCHED_CPUFREQ_IOWAIT) { 1769 cpu->iowait_boost = int_tofp(1);
1913 cpu->iowait_boost = int_tofp(1); 1770 } else if (cpu->iowait_boost) {
1914 } else if (cpu->iowait_boost) { 1771 /* Clear iowait_boost if the CPU may have been idle. */
1915 /* Clear iowait_boost if the CPU may have been idle. */ 1772 delta_ns = time - cpu->last_update;
1916 delta_ns = time - cpu->last_update; 1773 if (delta_ns > TICK_NSEC)
1917 if (delta_ns > TICK_NSEC) 1774 cpu->iowait_boost = 0;
1918 cpu->iowait_boost = 0;
1919 }
1920 cpu->last_update = time;
1921 } 1775 }
1922 1776 cpu->last_update = time;
1923 delta_ns = time - cpu->sample.time; 1777 delta_ns = time - cpu->sample.time;
1924 if ((s64)delta_ns >= pid_params.sample_rate_ns) { 1778 if ((s64)delta_ns < INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL)
1925 bool sample_taken = intel_pstate_sample(cpu, time); 1779 return;
1926 1780
1927 if (sample_taken) { 1781 if (intel_pstate_sample(cpu, time)) {
1928 intel_pstate_calc_avg_perf(cpu); 1782 int target_pstate;
1929 if (!hwp_active) 1783
1930 intel_pstate_adjust_busy_pstate(cpu); 1784 target_pstate = get_target_pstate_use_cpu_load(cpu);
1931 } 1785 intel_pstate_adjust_pstate(cpu, target_pstate);
1932 } 1786 }
1933} 1787}
1934 1788
1789static struct pstate_funcs core_funcs = {
1790 .get_max = core_get_max_pstate,
1791 .get_max_physical = core_get_max_pstate_physical,
1792 .get_min = core_get_min_pstate,
1793 .get_turbo = core_get_turbo_pstate,
1794 .get_scaling = core_get_scaling,
1795 .get_val = core_get_val,
1796 .update_util = intel_pstate_update_util_pid,
1797};
1798
1799static const struct pstate_funcs silvermont_funcs = {
1800 .get_max = atom_get_max_pstate,
1801 .get_max_physical = atom_get_max_pstate,
1802 .get_min = atom_get_min_pstate,
1803 .get_turbo = atom_get_turbo_pstate,
1804 .get_val = atom_get_val,
1805 .get_scaling = silvermont_get_scaling,
1806 .get_vid = atom_get_vid,
1807 .update_util = intel_pstate_update_util,
1808};
1809
1810static const struct pstate_funcs airmont_funcs = {
1811 .get_max = atom_get_max_pstate,
1812 .get_max_physical = atom_get_max_pstate,
1813 .get_min = atom_get_min_pstate,
1814 .get_turbo = atom_get_turbo_pstate,
1815 .get_val = atom_get_val,
1816 .get_scaling = airmont_get_scaling,
1817 .get_vid = atom_get_vid,
1818 .update_util = intel_pstate_update_util,
1819};
1820
1821static const struct pstate_funcs knl_funcs = {
1822 .get_max = core_get_max_pstate,
1823 .get_max_physical = core_get_max_pstate_physical,
1824 .get_min = core_get_min_pstate,
1825 .get_turbo = knl_get_turbo_pstate,
1826 .get_scaling = core_get_scaling,
1827 .get_val = core_get_val,
1828 .update_util = intel_pstate_update_util_pid,
1829};
1830
1831static const struct pstate_funcs bxt_funcs = {
1832 .get_max = core_get_max_pstate,
1833 .get_max_physical = core_get_max_pstate_physical,
1834 .get_min = core_get_min_pstate,
1835 .get_turbo = core_get_turbo_pstate,
1836 .get_scaling = core_get_scaling,
1837 .get_val = core_get_val,
1838 .update_util = intel_pstate_update_util,
1839};
1840
1935#define ICPU(model, policy) \ 1841#define ICPU(model, policy) \
1936 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\ 1842 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
1937 (unsigned long)&policy } 1843 (unsigned long)&policy }
1938 1844
1939static const struct x86_cpu_id intel_pstate_cpu_ids[] = { 1845static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1940 ICPU(INTEL_FAM6_SANDYBRIDGE, core_params), 1846 ICPU(INTEL_FAM6_SANDYBRIDGE, core_funcs),
1941 ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_params), 1847 ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_funcs),
1942 ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_params), 1848 ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_funcs),
1943 ICPU(INTEL_FAM6_IVYBRIDGE, core_params), 1849 ICPU(INTEL_FAM6_IVYBRIDGE, core_funcs),
1944 ICPU(INTEL_FAM6_HASWELL_CORE, core_params), 1850 ICPU(INTEL_FAM6_HASWELL_CORE, core_funcs),
1945 ICPU(INTEL_FAM6_BROADWELL_CORE, core_params), 1851 ICPU(INTEL_FAM6_BROADWELL_CORE, core_funcs),
1946 ICPU(INTEL_FAM6_IVYBRIDGE_X, core_params), 1852 ICPU(INTEL_FAM6_IVYBRIDGE_X, core_funcs),
1947 ICPU(INTEL_FAM6_HASWELL_X, core_params), 1853 ICPU(INTEL_FAM6_HASWELL_X, core_funcs),
1948 ICPU(INTEL_FAM6_HASWELL_ULT, core_params), 1854 ICPU(INTEL_FAM6_HASWELL_ULT, core_funcs),
1949 ICPU(INTEL_FAM6_HASWELL_GT3E, core_params), 1855 ICPU(INTEL_FAM6_HASWELL_GT3E, core_funcs),
1950 ICPU(INTEL_FAM6_BROADWELL_GT3E, core_params), 1856 ICPU(INTEL_FAM6_BROADWELL_GT3E, core_funcs),
1951 ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_params), 1857 ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_funcs),
1952 ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_params), 1858 ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_funcs),
1953 ICPU(INTEL_FAM6_BROADWELL_X, core_params), 1859 ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
1954 ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params), 1860 ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs),
1955 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), 1861 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
1956 ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params), 1862 ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_funcs),
1957 ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_params), 1863 ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_funcs),
1958 ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_params), 1864 ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_funcs),
1865 ICPU(INTEL_FAM6_ATOM_GEMINI_LAKE, bxt_funcs),
1959 {} 1866 {}
1960}; 1867};
1961MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); 1868MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
1962 1869
1963static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { 1870static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
1964 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), 1871 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
1965 ICPU(INTEL_FAM6_BROADWELL_X, core_params), 1872 ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
1966 ICPU(INTEL_FAM6_SKYLAKE_X, core_params), 1873 ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
1967 {} 1874 {}
1968}; 1875};
1969 1876
1970static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = { 1877static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
1971 ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_params), 1878 ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_funcs),
1972 {} 1879 {}
1973}; 1880};
1974 1881
1882static bool pid_in_use(void);
1883
1975static int intel_pstate_init_cpu(unsigned int cpunum) 1884static int intel_pstate_init_cpu(unsigned int cpunum)
1976{ 1885{
1977 struct cpudata *cpu; 1886 struct cpudata *cpu;
@@ -1979,18 +1888,11 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
1979 cpu = all_cpu_data[cpunum]; 1888 cpu = all_cpu_data[cpunum];
1980 1889
1981 if (!cpu) { 1890 if (!cpu) {
1982 unsigned int size = sizeof(struct cpudata); 1891 cpu = kzalloc(sizeof(*cpu), GFP_KERNEL);
1983
1984 if (per_cpu_limits)
1985 size += sizeof(struct perf_limits);
1986
1987 cpu = kzalloc(size, GFP_KERNEL);
1988 if (!cpu) 1892 if (!cpu)
1989 return -ENOMEM; 1893 return -ENOMEM;
1990 1894
1991 all_cpu_data[cpunum] = cpu; 1895 all_cpu_data[cpunum] = cpu;
1992 if (per_cpu_limits)
1993 cpu->perf_limits = (struct perf_limits *)(cpu + 1);
1994 1896
1995 cpu->epp_default = -EINVAL; 1897 cpu->epp_default = -EINVAL;
1996 cpu->epp_powersave = -EINVAL; 1898 cpu->epp_powersave = -EINVAL;
@@ -2009,14 +1911,12 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
2009 intel_pstate_disable_ee(cpunum); 1911 intel_pstate_disable_ee(cpunum);
2010 1912
2011 intel_pstate_hwp_enable(cpu); 1913 intel_pstate_hwp_enable(cpu);
2012 pid_params.sample_rate_ms = 50; 1914 } else if (pid_in_use()) {
2013 pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC; 1915 intel_pstate_pid_reset(cpu);
2014 } 1916 }
2015 1917
2016 intel_pstate_get_cpu_pstates(cpu); 1918 intel_pstate_get_cpu_pstates(cpu);
2017 1919
2018 intel_pstate_busy_pid_reset(cpu);
2019
2020 pr_debug("controlling: cpu %d\n", cpunum); 1920 pr_debug("controlling: cpu %d\n", cpunum);
2021 1921
2022 return 0; 1922 return 0;
@@ -2039,7 +1939,7 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
2039 /* Prevent intel_pstate_update_util() from using stale data. */ 1939 /* Prevent intel_pstate_update_util() from using stale data. */
2040 cpu->sample.time = 0; 1940 cpu->sample.time = 0;
2041 cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, 1941 cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
2042 intel_pstate_update_util); 1942 pstate_funcs.update_util);
2043 cpu->update_util_set = true; 1943 cpu->update_util_set = true;
2044} 1944}
2045 1945
@@ -2055,46 +1955,68 @@ static void intel_pstate_clear_update_util_hook(unsigned int cpu)
2055 synchronize_sched(); 1955 synchronize_sched();
2056} 1956}
2057 1957
1958static int intel_pstate_get_max_freq(struct cpudata *cpu)
1959{
1960 return global.turbo_disabled || global.no_turbo ?
1961 cpu->pstate.max_freq : cpu->pstate.turbo_freq;
1962}
1963
2058static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, 1964static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy,
2059 struct perf_limits *limits) 1965 struct cpudata *cpu)
2060{ 1966{
1967 int max_freq = intel_pstate_get_max_freq(cpu);
2061 int32_t max_policy_perf, min_policy_perf; 1968 int32_t max_policy_perf, min_policy_perf;
2062 1969
2063 max_policy_perf = div_ext_fp(policy->max, policy->cpuinfo.max_freq); 1970 max_policy_perf = div_ext_fp(policy->max, max_freq);
2064 max_policy_perf = clamp_t(int32_t, max_policy_perf, 0, int_ext_tofp(1)); 1971 max_policy_perf = clamp_t(int32_t, max_policy_perf, 0, int_ext_tofp(1));
2065 if (policy->max == policy->min) { 1972 if (policy->max == policy->min) {
2066 min_policy_perf = max_policy_perf; 1973 min_policy_perf = max_policy_perf;
2067 } else { 1974 } else {
2068 min_policy_perf = div_ext_fp(policy->min, 1975 min_policy_perf = div_ext_fp(policy->min, max_freq);
2069 policy->cpuinfo.max_freq);
2070 min_policy_perf = clamp_t(int32_t, min_policy_perf, 1976 min_policy_perf = clamp_t(int32_t, min_policy_perf,
2071 0, max_policy_perf); 1977 0, max_policy_perf);
2072 } 1978 }
2073 1979
2074 /* Normalize user input to [min_perf, max_perf] */ 1980 /* Normalize user input to [min_perf, max_perf] */
2075 limits->min_perf = max(min_policy_perf, 1981 if (per_cpu_limits) {
2076 percent_ext_fp(limits->min_sysfs_pct)); 1982 cpu->min_perf = min_policy_perf;
2077 limits->min_perf = min(limits->min_perf, max_policy_perf); 1983 cpu->max_perf = max_policy_perf;
2078 limits->max_perf = min(max_policy_perf, 1984 } else {
2079 percent_ext_fp(limits->max_sysfs_pct)); 1985 int32_t global_min, global_max;
2080 limits->max_perf = max(min_policy_perf, limits->max_perf); 1986
1987 /* Global limits are in percent of the maximum turbo P-state. */
1988 global_max = percent_ext_fp(global.max_perf_pct);
1989 global_min = percent_ext_fp(global.min_perf_pct);
1990 if (max_freq != cpu->pstate.turbo_freq) {
1991 int32_t turbo_factor;
1992
1993 turbo_factor = div_ext_fp(cpu->pstate.turbo_pstate,
1994 cpu->pstate.max_pstate);
1995 global_min = mul_ext_fp(global_min, turbo_factor);
1996 global_max = mul_ext_fp(global_max, turbo_factor);
1997 }
1998 global_min = clamp_t(int32_t, global_min, 0, global_max);
1999
2000 cpu->min_perf = max(min_policy_perf, global_min);
2001 cpu->min_perf = min(cpu->min_perf, max_policy_perf);
2002 cpu->max_perf = min(max_policy_perf, global_max);
2003 cpu->max_perf = max(min_policy_perf, cpu->max_perf);
2081 2004
2082 /* Make sure min_perf <= max_perf */ 2005 /* Make sure min_perf <= max_perf */
2083 limits->min_perf = min(limits->min_perf, limits->max_perf); 2006 cpu->min_perf = min(cpu->min_perf, cpu->max_perf);
2007 }
2084 2008
2085 limits->max_perf = round_up(limits->max_perf, EXT_FRAC_BITS); 2009 cpu->max_perf = round_up(cpu->max_perf, EXT_FRAC_BITS);
2086 limits->min_perf = round_up(limits->min_perf, EXT_FRAC_BITS); 2010 cpu->min_perf = round_up(cpu->min_perf, EXT_FRAC_BITS);
2087 limits->max_perf_pct = fp_ext_toint(limits->max_perf * 100);
2088 limits->min_perf_pct = fp_ext_toint(limits->min_perf * 100);
2089 2011
2090 pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu, 2012 pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu,
2091 limits->max_perf_pct, limits->min_perf_pct); 2013 fp_ext_toint(cpu->max_perf * 100),
2014 fp_ext_toint(cpu->min_perf * 100));
2092} 2015}
2093 2016
2094static int intel_pstate_set_policy(struct cpufreq_policy *policy) 2017static int intel_pstate_set_policy(struct cpufreq_policy *policy)
2095{ 2018{
2096 struct cpudata *cpu; 2019 struct cpudata *cpu;
2097 struct perf_limits *perf_limits = &global;
2098 2020
2099 if (!policy->cpuinfo.max_freq) 2021 if (!policy->cpuinfo.max_freq)
2100 return -ENODEV; 2022 return -ENODEV;
@@ -2105,19 +2027,9 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
2105 cpu = all_cpu_data[policy->cpu]; 2027 cpu = all_cpu_data[policy->cpu];
2106 cpu->policy = policy->policy; 2028 cpu->policy = policy->policy;
2107 2029
2108 if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
2109 policy->max < policy->cpuinfo.max_freq &&
2110 policy->max > cpu->pstate.max_pstate * cpu->pstate.scaling) {
2111 pr_debug("policy->max > max non turbo frequency\n");
2112 policy->max = policy->cpuinfo.max_freq;
2113 }
2114
2115 if (per_cpu_limits)
2116 perf_limits = cpu->perf_limits;
2117
2118 mutex_lock(&intel_pstate_limits_lock); 2030 mutex_lock(&intel_pstate_limits_lock);
2119 2031
2120 intel_pstate_update_perf_limits(policy, perf_limits); 2032 intel_pstate_update_perf_limits(policy, cpu);
2121 2033
2122 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { 2034 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
2123 /* 2035 /*
@@ -2130,38 +2042,38 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
2130 2042
2131 intel_pstate_set_update_util_hook(policy->cpu); 2043 intel_pstate_set_update_util_hook(policy->cpu);
2132 2044
2133 intel_pstate_hwp_set_policy(policy); 2045 if (hwp_active)
2046 intel_pstate_hwp_set(policy->cpu);
2134 2047
2135 mutex_unlock(&intel_pstate_limits_lock); 2048 mutex_unlock(&intel_pstate_limits_lock);
2136 2049
2137 return 0; 2050 return 0;
2138} 2051}
2139 2052
2053static void intel_pstate_adjust_policy_max(struct cpufreq_policy *policy,
2054 struct cpudata *cpu)
2055{
2056 if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
2057 policy->max < policy->cpuinfo.max_freq &&
2058 policy->max > cpu->pstate.max_freq) {
2059 pr_debug("policy->max > max non turbo frequency\n");
2060 policy->max = policy->cpuinfo.max_freq;
2061 }
2062}
2063
2140static int intel_pstate_verify_policy(struct cpufreq_policy *policy) 2064static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
2141{ 2065{
2142 struct cpudata *cpu = all_cpu_data[policy->cpu]; 2066 struct cpudata *cpu = all_cpu_data[policy->cpu];
2143 2067
2144 update_turbo_state(); 2068 update_turbo_state();
2145 policy->cpuinfo.max_freq = global.turbo_disabled || global.no_turbo ? 2069 cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
2146 cpu->pstate.max_freq : 2070 intel_pstate_get_max_freq(cpu));
2147 cpu->pstate.turbo_freq;
2148
2149 cpufreq_verify_within_cpu_limits(policy);
2150 2071
2151 if (policy->policy != CPUFREQ_POLICY_POWERSAVE && 2072 if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
2152 policy->policy != CPUFREQ_POLICY_PERFORMANCE) 2073 policy->policy != CPUFREQ_POLICY_PERFORMANCE)
2153 return -EINVAL; 2074 return -EINVAL;
2154 2075
2155 /* When per-CPU limits are used, sysfs limits are not used */ 2076 intel_pstate_adjust_policy_max(policy, cpu);
2156 if (!per_cpu_limits) {
2157 unsigned int max_freq, min_freq;
2158
2159 max_freq = policy->cpuinfo.max_freq *
2160 global.max_sysfs_pct / 100;
2161 min_freq = policy->cpuinfo.max_freq *
2162 global.min_sysfs_pct / 100;
2163 cpufreq_verify_within_limits(policy, min_freq, max_freq);
2164 }
2165 2077
2166 return 0; 2078 return 0;
2167} 2079}
@@ -2202,8 +2114,8 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
2202 2114
2203 cpu = all_cpu_data[policy->cpu]; 2115 cpu = all_cpu_data[policy->cpu];
2204 2116
2205 if (per_cpu_limits) 2117 cpu->max_perf = int_ext_tofp(1);
2206 intel_pstate_init_limits(cpu->perf_limits); 2118 cpu->min_perf = 0;
2207 2119
2208 policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling; 2120 policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
2209 policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling; 2121 policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
@@ -2257,10 +2169,12 @@ static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
2257 struct cpudata *cpu = all_cpu_data[policy->cpu]; 2169 struct cpudata *cpu = all_cpu_data[policy->cpu];
2258 2170
2259 update_turbo_state(); 2171 update_turbo_state();
2260 policy->cpuinfo.max_freq = global.no_turbo || global.turbo_disabled ? 2172 cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
2261 cpu->pstate.max_freq : cpu->pstate.turbo_freq; 2173 intel_pstate_get_max_freq(cpu));
2262 2174
2263 cpufreq_verify_within_cpu_limits(policy); 2175 intel_pstate_adjust_policy_max(policy, cpu);
2176
2177 intel_pstate_update_perf_limits(policy, cpu);
2264 2178
2265 return 0; 2179 return 0;
2266} 2180}
@@ -2324,6 +2238,7 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
2324 return ret; 2238 return ret;
2325 2239
2326 policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; 2240 policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
2241 policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
2327 /* This reflects the intel_pstate_get_cpu_pstates() setting. */ 2242 /* This reflects the intel_pstate_get_cpu_pstates() setting. */
2328 policy->cur = policy->cpuinfo.min_freq; 2243 policy->cur = policy->cpuinfo.min_freq;
2329 2244
@@ -2341,7 +2256,13 @@ static struct cpufreq_driver intel_cpufreq = {
2341 .name = "intel_cpufreq", 2256 .name = "intel_cpufreq",
2342}; 2257};
2343 2258
2344static struct cpufreq_driver *intel_pstate_driver = &intel_pstate; 2259static struct cpufreq_driver *default_driver = &intel_pstate;
2260
2261static bool pid_in_use(void)
2262{
2263 return intel_pstate_driver == &intel_pstate &&
2264 pstate_funcs.update_util == intel_pstate_update_util_pid;
2265}
2345 2266
2346static void intel_pstate_driver_cleanup(void) 2267static void intel_pstate_driver_cleanup(void)
2347{ 2268{
@@ -2358,26 +2279,26 @@ static void intel_pstate_driver_cleanup(void)
2358 } 2279 }
2359 } 2280 }
2360 put_online_cpus(); 2281 put_online_cpus();
2282 intel_pstate_driver = NULL;
2361} 2283}
2362 2284
2363static int intel_pstate_register_driver(void) 2285static int intel_pstate_register_driver(struct cpufreq_driver *driver)
2364{ 2286{
2365 int ret; 2287 int ret;
2366 2288
2367 intel_pstate_init_limits(&global); 2289 memset(&global, 0, sizeof(global));
2290 global.max_perf_pct = 100;
2368 2291
2292 intel_pstate_driver = driver;
2369 ret = cpufreq_register_driver(intel_pstate_driver); 2293 ret = cpufreq_register_driver(intel_pstate_driver);
2370 if (ret) { 2294 if (ret) {
2371 intel_pstate_driver_cleanup(); 2295 intel_pstate_driver_cleanup();
2372 return ret; 2296 return ret;
2373 } 2297 }
2374 2298
2375 mutex_lock(&intel_pstate_limits_lock); 2299 global.min_perf_pct = min_perf_pct_min();
2376 driver_registered = true;
2377 mutex_unlock(&intel_pstate_limits_lock);
2378 2300
2379 if (intel_pstate_driver == &intel_pstate && !hwp_active && 2301 if (pid_in_use())
2380 pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load)
2381 intel_pstate_debug_expose_params(); 2302 intel_pstate_debug_expose_params();
2382 2303
2383 return 0; 2304 return 0;
@@ -2388,14 +2309,9 @@ static int intel_pstate_unregister_driver(void)
2388 if (hwp_active) 2309 if (hwp_active)
2389 return -EBUSY; 2310 return -EBUSY;
2390 2311
2391 if (intel_pstate_driver == &intel_pstate && !hwp_active && 2312 if (pid_in_use())
2392 pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load)
2393 intel_pstate_debug_hide_params(); 2313 intel_pstate_debug_hide_params();
2394 2314
2395 mutex_lock(&intel_pstate_limits_lock);
2396 driver_registered = false;
2397 mutex_unlock(&intel_pstate_limits_lock);
2398
2399 cpufreq_unregister_driver(intel_pstate_driver); 2315 cpufreq_unregister_driver(intel_pstate_driver);
2400 intel_pstate_driver_cleanup(); 2316 intel_pstate_driver_cleanup();
2401 2317
@@ -2404,7 +2320,7 @@ static int intel_pstate_unregister_driver(void)
2404 2320
2405static ssize_t intel_pstate_show_status(char *buf) 2321static ssize_t intel_pstate_show_status(char *buf)
2406{ 2322{
2407 if (!driver_registered) 2323 if (!intel_pstate_driver)
2408 return sprintf(buf, "off\n"); 2324 return sprintf(buf, "off\n");
2409 2325
2410 return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ? 2326 return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ?
@@ -2416,11 +2332,11 @@ static int intel_pstate_update_status(const char *buf, size_t size)
2416 int ret; 2332 int ret;
2417 2333
2418 if (size == 3 && !strncmp(buf, "off", size)) 2334 if (size == 3 && !strncmp(buf, "off", size))
2419 return driver_registered ? 2335 return intel_pstate_driver ?
2420 intel_pstate_unregister_driver() : -EINVAL; 2336 intel_pstate_unregister_driver() : -EINVAL;
2421 2337
2422 if (size == 6 && !strncmp(buf, "active", size)) { 2338 if (size == 6 && !strncmp(buf, "active", size)) {
2423 if (driver_registered) { 2339 if (intel_pstate_driver) {
2424 if (intel_pstate_driver == &intel_pstate) 2340 if (intel_pstate_driver == &intel_pstate)
2425 return 0; 2341 return 0;
2426 2342
@@ -2429,13 +2345,12 @@ static int intel_pstate_update_status(const char *buf, size_t size)
2429 return ret; 2345 return ret;
2430 } 2346 }
2431 2347
2432 intel_pstate_driver = &intel_pstate; 2348 return intel_pstate_register_driver(&intel_pstate);
2433 return intel_pstate_register_driver();
2434 } 2349 }
2435 2350
2436 if (size == 7 && !strncmp(buf, "passive", size)) { 2351 if (size == 7 && !strncmp(buf, "passive", size)) {
2437 if (driver_registered) { 2352 if (intel_pstate_driver) {
2438 if (intel_pstate_driver != &intel_pstate) 2353 if (intel_pstate_driver == &intel_cpufreq)
2439 return 0; 2354 return 0;
2440 2355
2441 ret = intel_pstate_unregister_driver(); 2356 ret = intel_pstate_unregister_driver();
@@ -2443,8 +2358,7 @@ static int intel_pstate_update_status(const char *buf, size_t size)
2443 return ret; 2358 return ret;
2444 } 2359 }
2445 2360
2446 intel_pstate_driver = &intel_cpufreq; 2361 return intel_pstate_register_driver(&intel_cpufreq);
2447 return intel_pstate_register_driver();
2448 } 2362 }
2449 2363
2450 return -EINVAL; 2364 return -EINVAL;
@@ -2465,23 +2379,17 @@ static int __init intel_pstate_msrs_not_valid(void)
2465 return 0; 2379 return 0;
2466} 2380}
2467 2381
2468static void __init copy_pid_params(struct pstate_adjust_policy *policy)
2469{
2470 pid_params.sample_rate_ms = policy->sample_rate_ms;
2471 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
2472 pid_params.p_gain_pct = policy->p_gain_pct;
2473 pid_params.i_gain_pct = policy->i_gain_pct;
2474 pid_params.d_gain_pct = policy->d_gain_pct;
2475 pid_params.deadband = policy->deadband;
2476 pid_params.setpoint = policy->setpoint;
2477}
2478
2479#ifdef CONFIG_ACPI 2382#ifdef CONFIG_ACPI
2480static void intel_pstate_use_acpi_profile(void) 2383static void intel_pstate_use_acpi_profile(void)
2481{ 2384{
2482 if (acpi_gbl_FADT.preferred_profile == PM_MOBILE) 2385 switch (acpi_gbl_FADT.preferred_profile) {
2483 pstate_funcs.get_target_pstate = 2386 case PM_MOBILE:
2484 get_target_pstate_use_cpu_load; 2387 case PM_TABLET:
2388 case PM_APPLIANCE_PC:
2389 case PM_DESKTOP:
2390 case PM_WORKSTATION:
2391 pstate_funcs.update_util = intel_pstate_update_util;
2392 }
2485} 2393}
2486#else 2394#else
2487static void intel_pstate_use_acpi_profile(void) 2395static void intel_pstate_use_acpi_profile(void)
@@ -2498,7 +2406,7 @@ static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
2498 pstate_funcs.get_scaling = funcs->get_scaling; 2406 pstate_funcs.get_scaling = funcs->get_scaling;
2499 pstate_funcs.get_val = funcs->get_val; 2407 pstate_funcs.get_val = funcs->get_val;
2500 pstate_funcs.get_vid = funcs->get_vid; 2408 pstate_funcs.get_vid = funcs->get_vid;
2501 pstate_funcs.get_target_pstate = funcs->get_target_pstate; 2409 pstate_funcs.update_util = funcs->update_util;
2502 2410
2503 intel_pstate_use_acpi_profile(); 2411 intel_pstate_use_acpi_profile();
2504} 2412}
@@ -2637,28 +2545,30 @@ static const struct x86_cpu_id hwp_support_ids[] __initconst = {
2637 2545
2638static int __init intel_pstate_init(void) 2546static int __init intel_pstate_init(void)
2639{ 2547{
2640 const struct x86_cpu_id *id; 2548 int rc;
2641 struct cpu_defaults *cpu_def;
2642 int rc = 0;
2643 2549
2644 if (no_load) 2550 if (no_load)
2645 return -ENODEV; 2551 return -ENODEV;
2646 2552
2647 if (x86_match_cpu(hwp_support_ids) && !no_hwp) { 2553 if (x86_match_cpu(hwp_support_ids)) {
2648 copy_cpu_funcs(&core_params.funcs); 2554 copy_cpu_funcs(&core_funcs);
2649 hwp_active++; 2555 if (no_hwp) {
2650 intel_pstate.attr = hwp_cpufreq_attrs; 2556 pstate_funcs.update_util = intel_pstate_update_util;
2651 goto hwp_cpu_matched; 2557 } else {
2652 } 2558 hwp_active++;
2653 2559 intel_pstate.attr = hwp_cpufreq_attrs;
2654 id = x86_match_cpu(intel_pstate_cpu_ids); 2560 pstate_funcs.update_util = intel_pstate_update_util_hwp;
2655 if (!id) 2561 goto hwp_cpu_matched;
2656 return -ENODEV; 2562 }
2563 } else {
2564 const struct x86_cpu_id *id;
2657 2565
2658 cpu_def = (struct cpu_defaults *)id->driver_data; 2566 id = x86_match_cpu(intel_pstate_cpu_ids);
2567 if (!id)
2568 return -ENODEV;
2659 2569
2660 copy_pid_params(&cpu_def->pid_policy); 2570 copy_cpu_funcs((struct pstate_funcs *)id->driver_data);
2661 copy_cpu_funcs(&cpu_def->funcs); 2571 }
2662 2572
2663 if (intel_pstate_msrs_not_valid()) 2573 if (intel_pstate_msrs_not_valid())
2664 return -ENODEV; 2574 return -ENODEV;
@@ -2685,7 +2595,7 @@ hwp_cpu_matched:
2685 intel_pstate_sysfs_expose_params(); 2595 intel_pstate_sysfs_expose_params();
2686 2596
2687 mutex_lock(&intel_pstate_driver_lock); 2597 mutex_lock(&intel_pstate_driver_lock);
2688 rc = intel_pstate_register_driver(); 2598 rc = intel_pstate_register_driver(default_driver);
2689 mutex_unlock(&intel_pstate_driver_lock); 2599 mutex_unlock(&intel_pstate_driver_lock);
2690 if (rc) 2600 if (rc)
2691 return rc; 2601 return rc;
@@ -2706,7 +2616,7 @@ static int __init intel_pstate_setup(char *str)
2706 no_load = 1; 2616 no_load = 1;
2707 } else if (!strcmp(str, "passive")) { 2617 } else if (!strcmp(str, "passive")) {
2708 pr_info("Passive mode enabled\n"); 2618 pr_info("Passive mode enabled\n");
2709 intel_pstate_driver = &intel_cpufreq; 2619 default_driver = &intel_cpufreq;
2710 no_hwp = 1; 2620 no_hwp = 1;
2711 } 2621 }
2712 if (!strcmp(str, "no_hwp")) { 2622 if (!strcmp(str, "no_hwp")) {
diff --git a/drivers/cpufreq/mt8173-cpufreq.c b/drivers/cpufreq/mt8173-cpufreq.c
index ab25b1235a5e..fd1886faf33a 100644
--- a/drivers/cpufreq/mt8173-cpufreq.c
+++ b/drivers/cpufreq/mt8173-cpufreq.c
@@ -573,14 +573,33 @@ static struct platform_driver mt8173_cpufreq_platdrv = {
573 .probe = mt8173_cpufreq_probe, 573 .probe = mt8173_cpufreq_probe,
574}; 574};
575 575
576static int mt8173_cpufreq_driver_init(void) 576/* List of machines supported by this driver */
577static const struct of_device_id mt8173_cpufreq_machines[] __initconst = {
578 { .compatible = "mediatek,mt817x", },
579 { .compatible = "mediatek,mt8173", },
580 { .compatible = "mediatek,mt8176", },
581
582 { }
583};
584
585static int __init mt8173_cpufreq_driver_init(void)
577{ 586{
587 struct device_node *np;
588 const struct of_device_id *match;
578 struct platform_device *pdev; 589 struct platform_device *pdev;
579 int err; 590 int err;
580 591
581 if (!of_machine_is_compatible("mediatek,mt8173")) 592 np = of_find_node_by_path("/");
593 if (!np)
582 return -ENODEV; 594 return -ENODEV;
583 595
596 match = of_match_node(mt8173_cpufreq_machines, np);
597 of_node_put(np);
598 if (!match) {
599 pr_warn("Machine is not compatible with mt8173-cpufreq\n");
600 return -ENODEV;
601 }
602
584 err = platform_driver_register(&mt8173_cpufreq_platdrv); 603 err = platform_driver_register(&mt8173_cpufreq_platdrv);
585 if (err) 604 if (err)
586 return err; 605 return err;
diff --git a/drivers/cpufreq/qoriq-cpufreq.c b/drivers/cpufreq/qoriq-cpufreq.c
index bfec1bcd3835..e2ea433a5f9c 100644
--- a/drivers/cpufreq/qoriq-cpufreq.c
+++ b/drivers/cpufreq/qoriq-cpufreq.c
@@ -52,17 +52,27 @@ static u32 get_bus_freq(void)
52{ 52{
53 struct device_node *soc; 53 struct device_node *soc;
54 u32 sysfreq; 54 u32 sysfreq;
55 struct clk *pltclk;
56 int ret;
55 57
58 /* get platform freq by searching bus-frequency property */
56 soc = of_find_node_by_type(NULL, "soc"); 59 soc = of_find_node_by_type(NULL, "soc");
57 if (!soc) 60 if (soc) {
58 return 0; 61 ret = of_property_read_u32(soc, "bus-frequency", &sysfreq);
59 62 of_node_put(soc);
60 if (of_property_read_u32(soc, "bus-frequency", &sysfreq)) 63 if (!ret)
61 sysfreq = 0; 64 return sysfreq;
65 }
62 66
63 of_node_put(soc); 67 /* get platform freq by its clock name */
68 pltclk = clk_get(NULL, "cg-pll0-div1");
69 if (IS_ERR(pltclk)) {
70 pr_err("%s: can't get bus frequency %ld\n",
71 __func__, PTR_ERR(pltclk));
72 return PTR_ERR(pltclk);
73 }
64 74
65 return sysfreq; 75 return clk_get_rate(pltclk);
66} 76}
67 77
68static struct clk *cpu_to_clk(int cpu) 78static struct clk *cpu_to_clk(int cpu)
diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c
new file mode 100644
index 000000000000..fe7875311d62
--- /dev/null
+++ b/drivers/cpufreq/tegra186-cpufreq.c
@@ -0,0 +1,275 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 */
13
14#include <linux/cpufreq.h>
15#include <linux/dma-mapping.h>
16#include <linux/module.h>
17#include <linux/of.h>
18#include <linux/platform_device.h>
19
20#include <soc/tegra/bpmp.h>
21#include <soc/tegra/bpmp-abi.h>
22
23#define EDVD_CORE_VOLT_FREQ(core) (0x20 + (core) * 0x4)
24#define EDVD_CORE_VOLT_FREQ_F_SHIFT 0
25#define EDVD_CORE_VOLT_FREQ_V_SHIFT 16
26
27struct tegra186_cpufreq_cluster_info {
28 unsigned long offset;
29 int cpus[4];
30 unsigned int bpmp_cluster_id;
31};
32
33#define NO_CPU -1
34static const struct tegra186_cpufreq_cluster_info tegra186_clusters[] = {
35 /* Denver cluster */
36 {
37 .offset = SZ_64K * 7,
38 .cpus = { 1, 2, NO_CPU, NO_CPU },
39 .bpmp_cluster_id = 0,
40 },
41 /* A57 cluster */
42 {
43 .offset = SZ_64K * 6,
44 .cpus = { 0, 3, 4, 5 },
45 .bpmp_cluster_id = 1,
46 },
47};
48
49struct tegra186_cpufreq_cluster {
50 const struct tegra186_cpufreq_cluster_info *info;
51 struct cpufreq_frequency_table *table;
52};
53
54struct tegra186_cpufreq_data {
55 void __iomem *regs;
56
57 size_t num_clusters;
58 struct tegra186_cpufreq_cluster *clusters;
59};
60
61static int tegra186_cpufreq_init(struct cpufreq_policy *policy)
62{
63 struct tegra186_cpufreq_data *data = cpufreq_get_driver_data();
64 unsigned int i;
65
66 for (i = 0; i < data->num_clusters; i++) {
67 struct tegra186_cpufreq_cluster *cluster = &data->clusters[i];
68 const struct tegra186_cpufreq_cluster_info *info =
69 cluster->info;
70 int core;
71
72 for (core = 0; core < ARRAY_SIZE(info->cpus); core++) {
73 if (info->cpus[core] == policy->cpu)
74 break;
75 }
76 if (core == ARRAY_SIZE(info->cpus))
77 continue;
78
79 policy->driver_data =
80 data->regs + info->offset + EDVD_CORE_VOLT_FREQ(core);
81 cpufreq_table_validate_and_show(policy, cluster->table);
82 }
83
84 policy->cpuinfo.transition_latency = 300 * 1000;
85
86 return 0;
87}
88
89static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy,
90 unsigned int index)
91{
92 struct cpufreq_frequency_table *tbl = policy->freq_table + index;
93 void __iomem *edvd_reg = policy->driver_data;
94 u32 edvd_val = tbl->driver_data;
95
96 writel(edvd_val, edvd_reg);
97
98 return 0;
99}
100
101static struct cpufreq_driver tegra186_cpufreq_driver = {
102 .name = "tegra186",
103 .flags = CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY,
104 .verify = cpufreq_generic_frequency_table_verify,
105 .target_index = tegra186_cpufreq_set_target,
106 .init = tegra186_cpufreq_init,
107 .attr = cpufreq_generic_attr,
108};
109
110static struct cpufreq_frequency_table *init_vhint_table(
111 struct platform_device *pdev, struct tegra_bpmp *bpmp,
112 unsigned int cluster_id)
113{
114 struct cpufreq_frequency_table *table;
115 struct mrq_cpu_vhint_request req;
116 struct tegra_bpmp_message msg;
117 struct cpu_vhint_data *data;
118 int err, i, j, num_rates = 0;
119 dma_addr_t phys;
120 void *virt;
121
122 virt = dma_alloc_coherent(bpmp->dev, sizeof(*data), &phys,
123 GFP_KERNEL | GFP_DMA32);
124 if (!virt)
125 return ERR_PTR(-ENOMEM);
126
127 data = (struct cpu_vhint_data *)virt;
128
129 memset(&req, 0, sizeof(req));
130 req.addr = phys;
131 req.cluster_id = cluster_id;
132
133 memset(&msg, 0, sizeof(msg));
134 msg.mrq = MRQ_CPU_VHINT;
135 msg.tx.data = &req;
136 msg.tx.size = sizeof(req);
137
138 err = tegra_bpmp_transfer(bpmp, &msg);
139 if (err) {
140 table = ERR_PTR(err);
141 goto free;
142 }
143
144 for (i = data->vfloor; i <= data->vceil; i++) {
145 u16 ndiv = data->ndiv[i];
146
147 if (ndiv < data->ndiv_min || ndiv > data->ndiv_max)
148 continue;
149
150 /* Only store lowest voltage index for each rate */
151 if (i > 0 && ndiv == data->ndiv[i - 1])
152 continue;
153
154 num_rates++;
155 }
156
157 table = devm_kcalloc(&pdev->dev, num_rates + 1, sizeof(*table),
158 GFP_KERNEL);
159 if (!table) {
160 table = ERR_PTR(-ENOMEM);
161 goto free;
162 }
163
164 for (i = data->vfloor, j = 0; i <= data->vceil; i++) {
165 struct cpufreq_frequency_table *point;
166 u16 ndiv = data->ndiv[i];
167 u32 edvd_val = 0;
168
169 if (ndiv < data->ndiv_min || ndiv > data->ndiv_max)
170 continue;
171
172 /* Only store lowest voltage index for each rate */
173 if (i > 0 && ndiv == data->ndiv[i - 1])
174 continue;
175
176 edvd_val |= i << EDVD_CORE_VOLT_FREQ_V_SHIFT;
177 edvd_val |= ndiv << EDVD_CORE_VOLT_FREQ_F_SHIFT;
178
179 point = &table[j++];
180 point->driver_data = edvd_val;
181 point->frequency = data->ref_clk_hz * ndiv / data->pdiv /
182 data->mdiv / 1000;
183 }
184
185 table[j].frequency = CPUFREQ_TABLE_END;
186
187free:
188 dma_free_coherent(bpmp->dev, sizeof(*data), virt, phys);
189
190 return table;
191}
192
193static int tegra186_cpufreq_probe(struct platform_device *pdev)
194{
195 struct tegra186_cpufreq_data *data;
196 struct tegra_bpmp *bpmp;
197 struct resource *res;
198 unsigned int i = 0, err;
199
200 data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
201 if (!data)
202 return -ENOMEM;
203
204 data->clusters = devm_kcalloc(&pdev->dev, ARRAY_SIZE(tegra186_clusters),
205 sizeof(*data->clusters), GFP_KERNEL);
206 if (!data->clusters)
207 return -ENOMEM;
208
209 data->num_clusters = ARRAY_SIZE(tegra186_clusters);
210
211 bpmp = tegra_bpmp_get(&pdev->dev);
212 if (IS_ERR(bpmp))
213 return PTR_ERR(bpmp);
214
215 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
216 data->regs = devm_ioremap_resource(&pdev->dev, res);
217 if (IS_ERR(data->regs)) {
218 err = PTR_ERR(data->regs);
219 goto put_bpmp;
220 }
221
222 for (i = 0; i < data->num_clusters; i++) {
223 struct tegra186_cpufreq_cluster *cluster = &data->clusters[i];
224
225 cluster->info = &tegra186_clusters[i];
226 cluster->table = init_vhint_table(
227 pdev, bpmp, cluster->info->bpmp_cluster_id);
228 if (IS_ERR(cluster->table)) {
229 err = PTR_ERR(cluster->table);
230 goto put_bpmp;
231 }
232 }
233
234 tegra_bpmp_put(bpmp);
235
236 tegra186_cpufreq_driver.driver_data = data;
237
238 err = cpufreq_register_driver(&tegra186_cpufreq_driver);
239 if (err)
240 return err;
241
242 return 0;
243
244put_bpmp:
245 tegra_bpmp_put(bpmp);
246
247 return err;
248}
249
250static int tegra186_cpufreq_remove(struct platform_device *pdev)
251{
252 cpufreq_unregister_driver(&tegra186_cpufreq_driver);
253
254 return 0;
255}
256
257static const struct of_device_id tegra186_cpufreq_of_match[] = {
258 { .compatible = "nvidia,tegra186-ccplex-cluster", },
259 { }
260};
261MODULE_DEVICE_TABLE(of, tegra186_cpufreq_of_match);
262
263static struct platform_driver tegra186_cpufreq_platform_driver = {
264 .driver = {
265 .name = "tegra186-cpufreq",
266 .of_match_table = tegra186_cpufreq_of_match,
267 },
268 .probe = tegra186_cpufreq_probe,
269 .remove = tegra186_cpufreq_remove,
270};
271module_platform_driver(tegra186_cpufreq_platform_driver);
272
273MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>");
274MODULE_DESCRIPTION("NVIDIA Tegra186 cpufreq driver");
275MODULE_LICENSE("GPL v2");
diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig
index 776b34396144..0a16cf4bed39 100644
--- a/drivers/thermal/Kconfig
+++ b/drivers/thermal/Kconfig
@@ -291,18 +291,6 @@ config ARMADA_THERMAL
291 Enable this option if you want to have support for thermal management 291 Enable this option if you want to have support for thermal management
292 controller present in Armada 370 and Armada XP SoC. 292 controller present in Armada 370 and Armada XP SoC.
293 293
294config DB8500_CPUFREQ_COOLING
295 tristate "DB8500 cpufreq cooling"
296 depends on ARCH_U8500 || COMPILE_TEST
297 depends on HAS_IOMEM
298 depends on CPU_THERMAL
299 default y
300 help
301 Adds DB8500 cpufreq cooling devices, and these cooling devices can be
302 bound to thermal zone trip points. When a trip point reached, the
303 bound cpufreq cooling device turns active to set CPU frequency low to
304 cool down the CPU.
305
306config INTEL_POWERCLAMP 294config INTEL_POWERCLAMP
307 tristate "Intel PowerClamp idle injection driver" 295 tristate "Intel PowerClamp idle injection driver"
308 depends on THERMAL 296 depends on THERMAL
diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile
index 7adae2029355..c2372f10dae5 100644
--- a/drivers/thermal/Makefile
+++ b/drivers/thermal/Makefile
@@ -41,7 +41,6 @@ obj-$(CONFIG_TANGO_THERMAL) += tango_thermal.o
41obj-$(CONFIG_IMX_THERMAL) += imx_thermal.o 41obj-$(CONFIG_IMX_THERMAL) += imx_thermal.o
42obj-$(CONFIG_MAX77620_THERMAL) += max77620_thermal.o 42obj-$(CONFIG_MAX77620_THERMAL) += max77620_thermal.o
43obj-$(CONFIG_QORIQ_THERMAL) += qoriq_thermal.o 43obj-$(CONFIG_QORIQ_THERMAL) += qoriq_thermal.o
44obj-$(CONFIG_DB8500_CPUFREQ_COOLING) += db8500_cpufreq_cooling.o
45obj-$(CONFIG_INTEL_POWERCLAMP) += intel_powerclamp.o 44obj-$(CONFIG_INTEL_POWERCLAMP) += intel_powerclamp.o
46obj-$(CONFIG_X86_PKG_TEMP_THERMAL) += x86_pkg_temp_thermal.o 45obj-$(CONFIG_X86_PKG_TEMP_THERMAL) += x86_pkg_temp_thermal.o
47obj-$(CONFIG_INTEL_SOC_DTS_IOSF_CORE) += intel_soc_dts_iosf.o 46obj-$(CONFIG_INTEL_SOC_DTS_IOSF_CORE) += intel_soc_dts_iosf.o
diff --git a/drivers/thermal/db8500_cpufreq_cooling.c b/drivers/thermal/db8500_cpufreq_cooling.c
deleted file mode 100644
index e58bd0b658b5..000000000000
--- a/drivers/thermal/db8500_cpufreq_cooling.c
+++ /dev/null
@@ -1,105 +0,0 @@
1/*
2 * db8500_cpufreq_cooling.c - DB8500 cpufreq works as cooling device.
3 *
4 * Copyright (C) 2012 ST-Ericsson
5 * Copyright (C) 2012 Linaro Ltd.
6 *
7 * Author: Hongbo Zhang <hongbo.zhang@linaro.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 */
19
20#include <linux/cpu_cooling.h>
21#include <linux/err.h>
22#include <linux/module.h>
23#include <linux/of.h>
24#include <linux/platform_device.h>
25#include <linux/slab.h>
26
27static int db8500_cpufreq_cooling_probe(struct platform_device *pdev)
28{
29 struct thermal_cooling_device *cdev;
30
31 cdev = cpufreq_cooling_register(cpu_present_mask);
32 if (IS_ERR(cdev)) {
33 int ret = PTR_ERR(cdev);
34
35 if (ret != -EPROBE_DEFER)
36 dev_err(&pdev->dev,
37 "Failed to register cooling device %d\n",
38 ret);
39
40 return ret;
41 }
42
43 platform_set_drvdata(pdev, cdev);
44
45 dev_info(&pdev->dev, "Cooling device registered: %s\n", cdev->type);
46
47 return 0;
48}
49
50static int db8500_cpufreq_cooling_remove(struct platform_device *pdev)
51{
52 struct thermal_cooling_device *cdev = platform_get_drvdata(pdev);
53
54 cpufreq_cooling_unregister(cdev);
55
56 return 0;
57}
58
59static int db8500_cpufreq_cooling_suspend(struct platform_device *pdev,
60 pm_message_t state)
61{
62 return -ENOSYS;
63}
64
65static int db8500_cpufreq_cooling_resume(struct platform_device *pdev)
66{
67 return -ENOSYS;
68}
69
70#ifdef CONFIG_OF
71static const struct of_device_id db8500_cpufreq_cooling_match[] = {
72 { .compatible = "stericsson,db8500-cpufreq-cooling" },
73 {},
74};
75MODULE_DEVICE_TABLE(of, db8500_cpufreq_cooling_match);
76#endif
77
78static struct platform_driver db8500_cpufreq_cooling_driver = {
79 .driver = {
80 .name = "db8500-cpufreq-cooling",
81 .of_match_table = of_match_ptr(db8500_cpufreq_cooling_match),
82 },
83 .probe = db8500_cpufreq_cooling_probe,
84 .suspend = db8500_cpufreq_cooling_suspend,
85 .resume = db8500_cpufreq_cooling_resume,
86 .remove = db8500_cpufreq_cooling_remove,
87};
88
89static int __init db8500_cpufreq_cooling_init(void)
90{
91 return platform_driver_register(&db8500_cpufreq_cooling_driver);
92}
93
94static void __exit db8500_cpufreq_cooling_exit(void)
95{
96 platform_driver_unregister(&db8500_cpufreq_cooling_driver);
97}
98
99/* Should be later than db8500_cpufreq_register */
100late_initcall(db8500_cpufreq_cooling_init);
101module_exit(db8500_cpufreq_cooling_exit);
102
103MODULE_AUTHOR("Hongbo Zhang <hongbo.zhang@stericsson.com>");
104MODULE_DESCRIPTION("DB8500 cpufreq cooling driver");
105MODULE_LICENSE("GPL");
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index 87165f06a307..a5ce0bbeadb5 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -120,6 +120,13 @@ struct cpufreq_policy {
120 bool fast_switch_possible; 120 bool fast_switch_possible;
121 bool fast_switch_enabled; 121 bool fast_switch_enabled;
122 122
123 /*
124 * Preferred average time interval between consecutive invocations of
125 * the driver to set the frequency for this policy. To be set by the
126 * scaling driver (0, which is the default, means no preference).
127 */
128 unsigned int transition_delay_us;
129
123 /* Cached frequency lookup from cpufreq_driver_resolve_freq. */ 130 /* Cached frequency lookup from cpufreq_driver_resolve_freq. */
124 unsigned int cached_target_freq; 131 unsigned int cached_target_freq;
125 int cached_resolved_idx; 132 int cached_resolved_idx;
diff --git a/include/linux/tick.h b/include/linux/tick.h
index a04fea19676f..fe01e68bf520 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -117,6 +117,7 @@ extern void tick_nohz_idle_enter(void);
117extern void tick_nohz_idle_exit(void); 117extern void tick_nohz_idle_exit(void);
118extern void tick_nohz_irq_exit(void); 118extern void tick_nohz_irq_exit(void);
119extern ktime_t tick_nohz_get_sleep_length(void); 119extern ktime_t tick_nohz_get_sleep_length(void);
120extern unsigned long tick_nohz_get_idle_calls(void);
120extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); 121extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
121extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); 122extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
122#else /* !CONFIG_NO_HZ_COMMON */ 123#else /* !CONFIG_NO_HZ_COMMON */
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index 54c577578da6..76877a62b5fa 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -61,6 +61,11 @@ struct sugov_cpu {
61 unsigned long util; 61 unsigned long util;
62 unsigned long max; 62 unsigned long max;
63 unsigned int flags; 63 unsigned int flags;
64
65 /* The field below is for single-CPU policies only. */
66#ifdef CONFIG_NO_HZ_COMMON
67 unsigned long saved_idle_calls;
68#endif
64}; 69};
65 70
66static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu); 71static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);
@@ -93,22 +98,23 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
93{ 98{
94 struct cpufreq_policy *policy = sg_policy->policy; 99 struct cpufreq_policy *policy = sg_policy->policy;
95 100
101 if (sg_policy->next_freq == next_freq)
102 return;
103
104 if (sg_policy->next_freq > next_freq)
105 next_freq = (sg_policy->next_freq + next_freq) >> 1;
106
107 sg_policy->next_freq = next_freq;
96 sg_policy->last_freq_update_time = time; 108 sg_policy->last_freq_update_time = time;
97 109
98 if (policy->fast_switch_enabled) { 110 if (policy->fast_switch_enabled) {
99 if (sg_policy->next_freq == next_freq) {
100 trace_cpu_frequency(policy->cur, smp_processor_id());
101 return;
102 }
103 sg_policy->next_freq = next_freq;
104 next_freq = cpufreq_driver_fast_switch(policy, next_freq); 111 next_freq = cpufreq_driver_fast_switch(policy, next_freq);
105 if (next_freq == CPUFREQ_ENTRY_INVALID) 112 if (next_freq == CPUFREQ_ENTRY_INVALID)
106 return; 113 return;
107 114
108 policy->cur = next_freq; 115 policy->cur = next_freq;
109 trace_cpu_frequency(next_freq, smp_processor_id()); 116 trace_cpu_frequency(next_freq, smp_processor_id());
110 } else if (sg_policy->next_freq != next_freq) { 117 } else {
111 sg_policy->next_freq = next_freq;
112 sg_policy->work_in_progress = true; 118 sg_policy->work_in_progress = true;
113 irq_work_queue(&sg_policy->irq_work); 119 irq_work_queue(&sg_policy->irq_work);
114 } 120 }
@@ -192,6 +198,19 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, unsigned long *util,
192 sg_cpu->iowait_boost >>= 1; 198 sg_cpu->iowait_boost >>= 1;
193} 199}
194 200
201#ifdef CONFIG_NO_HZ_COMMON
202static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)
203{
204 unsigned long idle_calls = tick_nohz_get_idle_calls();
205 bool ret = idle_calls == sg_cpu->saved_idle_calls;
206
207 sg_cpu->saved_idle_calls = idle_calls;
208 return ret;
209}
210#else
211static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }
212#endif /* CONFIG_NO_HZ_COMMON */
213
195static void sugov_update_single(struct update_util_data *hook, u64 time, 214static void sugov_update_single(struct update_util_data *hook, u64 time,
196 unsigned int flags) 215 unsigned int flags)
197{ 216{
@@ -200,6 +219,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
200 struct cpufreq_policy *policy = sg_policy->policy; 219 struct cpufreq_policy *policy = sg_policy->policy;
201 unsigned long util, max; 220 unsigned long util, max;
202 unsigned int next_f; 221 unsigned int next_f;
222 bool busy;
203 223
204 sugov_set_iowait_boost(sg_cpu, time, flags); 224 sugov_set_iowait_boost(sg_cpu, time, flags);
205 sg_cpu->last_update = time; 225 sg_cpu->last_update = time;
@@ -207,40 +227,37 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
207 if (!sugov_should_update_freq(sg_policy, time)) 227 if (!sugov_should_update_freq(sg_policy, time))
208 return; 228 return;
209 229
230 busy = sugov_cpu_is_busy(sg_cpu);
231
210 if (flags & SCHED_CPUFREQ_RT_DL) { 232 if (flags & SCHED_CPUFREQ_RT_DL) {
211 next_f = policy->cpuinfo.max_freq; 233 next_f = policy->cpuinfo.max_freq;
212 } else { 234 } else {
213 sugov_get_util(&util, &max); 235 sugov_get_util(&util, &max);
214 sugov_iowait_boost(sg_cpu, &util, &max); 236 sugov_iowait_boost(sg_cpu, &util, &max);
215 next_f = get_next_freq(sg_policy, util, max); 237 next_f = get_next_freq(sg_policy, util, max);
238 /*
239 * Do not reduce the frequency if the CPU has not been idle
240 * recently, as the reduction is likely to be premature then.
241 */
242 if (busy && next_f < sg_policy->next_freq)
243 next_f = sg_policy->next_freq;
216 } 244 }
217 sugov_update_commit(sg_policy, time, next_f); 245 sugov_update_commit(sg_policy, time, next_f);
218} 246}
219 247
220static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, 248static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu)
221 unsigned long util, unsigned long max,
222 unsigned int flags)
223{ 249{
224 struct sugov_policy *sg_policy = sg_cpu->sg_policy; 250 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
225 struct cpufreq_policy *policy = sg_policy->policy; 251 struct cpufreq_policy *policy = sg_policy->policy;
226 unsigned int max_f = policy->cpuinfo.max_freq;
227 u64 last_freq_update_time = sg_policy->last_freq_update_time; 252 u64 last_freq_update_time = sg_policy->last_freq_update_time;
253 unsigned long util = 0, max = 1;
228 unsigned int j; 254 unsigned int j;
229 255
230 if (flags & SCHED_CPUFREQ_RT_DL)
231 return max_f;
232
233 sugov_iowait_boost(sg_cpu, &util, &max);
234
235 for_each_cpu(j, policy->cpus) { 256 for_each_cpu(j, policy->cpus) {
236 struct sugov_cpu *j_sg_cpu; 257 struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j);
237 unsigned long j_util, j_max; 258 unsigned long j_util, j_max;
238 s64 delta_ns; 259 s64 delta_ns;
239 260
240 if (j == smp_processor_id())
241 continue;
242
243 j_sg_cpu = &per_cpu(sugov_cpu, j);
244 /* 261 /*
245 * If the CPU utilization was last updated before the previous 262 * If the CPU utilization was last updated before the previous
246 * frequency update and the time elapsed between the last update 263 * frequency update and the time elapsed between the last update
@@ -254,7 +271,7 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
254 continue; 271 continue;
255 } 272 }
256 if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL) 273 if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL)
257 return max_f; 274 return policy->cpuinfo.max_freq;
258 275
259 j_util = j_sg_cpu->util; 276 j_util = j_sg_cpu->util;
260 j_max = j_sg_cpu->max; 277 j_max = j_sg_cpu->max;
@@ -289,7 +306,11 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
289 sg_cpu->last_update = time; 306 sg_cpu->last_update = time;
290 307
291 if (sugov_should_update_freq(sg_policy, time)) { 308 if (sugov_should_update_freq(sg_policy, time)) {
292 next_f = sugov_next_freq_shared(sg_cpu, util, max, flags); 309 if (flags & SCHED_CPUFREQ_RT_DL)
310 next_f = sg_policy->policy->cpuinfo.max_freq;
311 else
312 next_f = sugov_next_freq_shared(sg_cpu);
313
293 sugov_update_commit(sg_policy, time, next_f); 314 sugov_update_commit(sg_policy, time, next_f);
294 } 315 }
295 316
@@ -473,7 +494,6 @@ static int sugov_init(struct cpufreq_policy *policy)
473{ 494{
474 struct sugov_policy *sg_policy; 495 struct sugov_policy *sg_policy;
475 struct sugov_tunables *tunables; 496 struct sugov_tunables *tunables;
476 unsigned int lat;
477 int ret = 0; 497 int ret = 0;
478 498
479 /* State should be equivalent to EXIT */ 499 /* State should be equivalent to EXIT */
@@ -512,10 +532,16 @@ static int sugov_init(struct cpufreq_policy *policy)
512 goto stop_kthread; 532 goto stop_kthread;
513 } 533 }
514 534
515 tunables->rate_limit_us = LATENCY_MULTIPLIER; 535 if (policy->transition_delay_us) {
516 lat = policy->cpuinfo.transition_latency / NSEC_PER_USEC; 536 tunables->rate_limit_us = policy->transition_delay_us;
517 if (lat) 537 } else {
518 tunables->rate_limit_us *= lat; 538 unsigned int lat;
539
540 tunables->rate_limit_us = LATENCY_MULTIPLIER;
541 lat = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
542 if (lat)
543 tunables->rate_limit_us *= lat;
544 }
519 545
520 policy->governor_data = sg_policy; 546 policy->governor_data = sg_policy;
521 sg_policy->tunables = tunables; 547 sg_policy->tunables = tunables;
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 7fe53be86077..64c97fc130c4 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -993,6 +993,18 @@ ktime_t tick_nohz_get_sleep_length(void)
993 return ts->sleep_length; 993 return ts->sleep_length;
994} 994}
995 995
996/**
997 * tick_nohz_get_idle_calls - return the current idle calls counter value
998 *
999 * Called from the schedutil frequency scaling governor in scheduler context.
1000 */
1001unsigned long tick_nohz_get_idle_calls(void)
1002{
1003 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1004
1005 return ts->idle_calls;
1006}
1007
996static void tick_nohz_account_idle_ticks(struct tick_sched *ts) 1008static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
997{ 1009{
998#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 1010#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
generated by cgit v1.2.2 (git 2.25.0) at 2026-07-08 03:54:31 -0400