aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/cpufreq/intel_pstate.c1021
1 files changed, 431 insertions, 590 deletions
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 08e134ffba68..c31b72b16c2b 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -37,6 +37,9 @@
37#include <asm/cpufeature.h> 37#include <asm/cpufeature.h>
38#include <asm/intel-family.h> 38#include <asm/intel-family.h>
39 39
40#define INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
41#define INTEL_PSTATE_HWP_SAMPLING_INTERVAL (50 * NSEC_PER_MSEC)
42
40#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 43#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
41 44
42#ifdef CONFIG_ACPI 45#ifdef CONFIG_ACPI
@@ -74,6 +77,11 @@ static inline int ceiling_fp(int32_t x)
74 return ret; 77 return ret;
75} 78}
76 79
80static inline int32_t percent_fp(int percent)
81{
82 return div_fp(percent, 100);
83}
84
77static inline u64 mul_ext_fp(u64 x, u64 y) 85static inline u64 mul_ext_fp(u64 x, u64 y)
78{ 86{
79 return (x * y) >> EXT_FRAC_BITS; 87 return (x * y) >> EXT_FRAC_BITS;
@@ -186,45 +194,22 @@ struct _pid {
186}; 194};
187 195
188/** 196/**
189 * struct perf_limits - Store user and policy limits 197 * struct global_params - Global parameters, mostly tunable via sysfs.
190 * @no_turbo: User requested turbo state from intel_pstate sysfs 198 * @no_turbo: Whether or not to use turbo P-states.
191 * @turbo_disabled: Platform turbo status either from msr 199 * @turbo_disabled: Whethet or not turbo P-states are available at all,
192 * MSR_IA32_MISC_ENABLE or when maximum available pstate 200 * based on the MSR_IA32_MISC_ENABLE value and whether or
193 * matches the maximum turbo pstate 201 * not the maximum reported turbo P-state is different from
194 * @max_perf_pct: Effective maximum performance limit in percentage, this 202 * the maximum reported non-turbo one.
195 * is minimum of either limits enforced by cpufreq policy 203 * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
196 * or limits from user set limits via intel_pstate sysfs 204 * P-state capacity.
197 * @min_perf_pct: Effective minimum performance limit in percentage, this 205 * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
198 * is maximum of either limits enforced by cpufreq policy 206 * P-state capacity.
199 * or limits from user set limits via intel_pstate sysfs
200 * @max_perf: This is a scaled value between 0 to 255 for max_perf_pct
201 * This value is used to limit max pstate
202 * @min_perf: This is a scaled value between 0 to 255 for min_perf_pct
203 * This value is used to limit min pstate
204 * @max_policy_pct: The maximum performance in percentage enforced by
205 * cpufreq setpolicy interface
206 * @max_sysfs_pct: The maximum performance in percentage enforced by
207 * intel pstate sysfs interface, unused when per cpu
208 * controls are enforced
209 * @min_policy_pct: The minimum performance in percentage enforced by
210 * cpufreq setpolicy interface
211 * @min_sysfs_pct: The minimum performance in percentage enforced by
212 * intel pstate sysfs interface, unused when per cpu
213 * controls are enforced
214 *
215 * Storage for user and policy defined limits.
216 */ 207 */
217struct perf_limits { 208struct global_params {
218 int no_turbo; 209 bool no_turbo;
219 int turbo_disabled; 210 bool turbo_disabled;
220 int max_perf_pct; 211 int max_perf_pct;
221 int min_perf_pct; 212 int min_perf_pct;
222 int32_t max_perf;
223 int32_t min_perf;
224 int max_policy_pct;
225 int max_sysfs_pct;
226 int min_policy_pct;
227 int min_sysfs_pct;
228}; 213};
229 214
230/** 215/**
@@ -245,9 +230,10 @@ struct perf_limits {
245 * @prev_cummulative_iowait: IO Wait time difference from last and 230 * @prev_cummulative_iowait: IO Wait time difference from last and
246 * current sample 231 * current sample
247 * @sample: Storage for storing last Sample data 232 * @sample: Storage for storing last Sample data
248 * @perf_limits: Pointer to perf_limit unique to this CPU 233 * @min_perf: Minimum capacity limit as a fraction of the maximum
249 * Not all field in the structure are applicable 234 * turbo P-state capacity.
250 * when per cpu controls are enforced 235 * @max_perf: Maximum capacity limit as a fraction of the maximum
236 * turbo P-state capacity.
251 * @acpi_perf_data: Stores ACPI perf information read from _PSS 237 * @acpi_perf_data: Stores ACPI perf information read from _PSS
252 * @valid_pss_table: Set to true for valid ACPI _PSS entries found 238 * @valid_pss_table: Set to true for valid ACPI _PSS entries found
253 * @epp_powersave: Last saved HWP energy performance preference 239 * @epp_powersave: Last saved HWP energy performance preference
@@ -279,7 +265,8 @@ struct cpudata {
279 u64 prev_tsc; 265 u64 prev_tsc;
280 u64 prev_cummulative_iowait; 266 u64 prev_cummulative_iowait;
281 struct sample sample; 267 struct sample sample;
282 struct perf_limits *perf_limits; 268 int32_t min_perf;
269 int32_t max_perf;
283#ifdef CONFIG_ACPI 270#ifdef CONFIG_ACPI
284 struct acpi_processor_performance acpi_perf_data; 271 struct acpi_processor_performance acpi_perf_data;
285 bool valid_pss_table; 272 bool valid_pss_table;
@@ -324,7 +311,7 @@ struct pstate_adjust_policy {
324 * @get_scaling: Callback to get frequency scaling factor 311 * @get_scaling: Callback to get frequency scaling factor
325 * @get_val: Callback to convert P state to actual MSR write value 312 * @get_val: Callback to convert P state to actual MSR write value
326 * @get_vid: Callback to get VID data for Atom platforms 313 * @get_vid: Callback to get VID data for Atom platforms
327 * @get_target_pstate: Callback to a function to calculate next P state to use 314 * @update_util: Active mode utilization update callback.
328 * 315 *
329 * Core and Atom CPU models have different way to get P State limits. This 316 * Core and Atom CPU models have different way to get P State limits. This
330 * structure is used to store those callbacks. 317 * structure is used to store those callbacks.
@@ -337,53 +324,31 @@ struct pstate_funcs {
337 int (*get_scaling)(void); 324 int (*get_scaling)(void);
338 u64 (*get_val)(struct cpudata*, int pstate); 325 u64 (*get_val)(struct cpudata*, int pstate);
339 void (*get_vid)(struct cpudata *); 326 void (*get_vid)(struct cpudata *);
340 int32_t (*get_target_pstate)(struct cpudata *); 327 void (*update_util)(struct update_util_data *data, u64 time,
328 unsigned int flags);
341}; 329};
342 330
343/** 331static struct pstate_funcs pstate_funcs __read_mostly;
344 * struct cpu_defaults- Per CPU model default config data 332static struct pstate_adjust_policy pid_params __read_mostly = {
345 * @pid_policy: PID config data 333 .sample_rate_ms = 10,
346 * @funcs: Callback function data 334 .sample_rate_ns = 10 * NSEC_PER_MSEC,
347 */ 335 .deadband = 0,
348struct cpu_defaults { 336 .setpoint = 97,
349 struct pstate_adjust_policy pid_policy; 337 .p_gain_pct = 20,
350 struct pstate_funcs funcs; 338 .d_gain_pct = 0,
339 .i_gain_pct = 0,
351}; 340};
352 341
353static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu);
354static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu);
355
356static struct pstate_adjust_policy pid_params __read_mostly;
357static struct pstate_funcs pstate_funcs __read_mostly;
358static int hwp_active __read_mostly; 342static int hwp_active __read_mostly;
359static bool per_cpu_limits __read_mostly; 343static bool per_cpu_limits __read_mostly;
360 344
361static bool driver_registered __read_mostly; 345static struct cpufreq_driver *intel_pstate_driver __read_mostly;
362 346
363#ifdef CONFIG_ACPI 347#ifdef CONFIG_ACPI
364static bool acpi_ppc; 348static bool acpi_ppc;
365#endif 349#endif
366 350
367static struct perf_limits performance_limits; 351static struct global_params global;
368static struct perf_limits powersave_limits;
369static struct perf_limits *limits;
370
371static void intel_pstate_init_limits(struct perf_limits *limits)
372{
373 memset(limits, 0, sizeof(*limits));
374 limits->max_perf_pct = 100;
375 limits->max_perf = int_ext_tofp(1);
376 limits->max_policy_pct = 100;
377 limits->max_sysfs_pct = 100;
378}
379
380static void intel_pstate_set_performance_limits(struct perf_limits *limits)
381{
382 intel_pstate_init_limits(limits);
383 limits->min_perf_pct = 100;
384 limits->min_perf = int_ext_tofp(1);
385 limits->min_sysfs_pct = 100;
386}
387 352
388static DEFINE_MUTEX(intel_pstate_driver_lock); 353static DEFINE_MUTEX(intel_pstate_driver_lock);
389static DEFINE_MUTEX(intel_pstate_limits_lock); 354static DEFINE_MUTEX(intel_pstate_limits_lock);
@@ -507,7 +472,7 @@ static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
507 * correct max turbo frequency based on the turbo state. 472 * correct max turbo frequency based on the turbo state.
508 * Also need to convert to MHz as _PSS freq is in MHz. 473 * Also need to convert to MHz as _PSS freq is in MHz.
509 */ 474 */
510 if (!limits->turbo_disabled) 475 if (!global.turbo_disabled)
511 cpu->acpi_perf_data.states[0].core_frequency = 476 cpu->acpi_perf_data.states[0].core_frequency =
512 policy->cpuinfo.max_freq / 1000; 477 policy->cpuinfo.max_freq / 1000;
513 cpu->valid_pss_table = true; 478 cpu->valid_pss_table = true;
@@ -540,29 +505,6 @@ static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
540} 505}
541#endif 506#endif
542 507
543static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
544 int deadband, int integral) {
545 pid->setpoint = int_tofp(setpoint);
546 pid->deadband = int_tofp(deadband);
547 pid->integral = int_tofp(integral);
548 pid->last_err = int_tofp(setpoint) - int_tofp(busy);
549}
550
551static inline void pid_p_gain_set(struct _pid *pid, int percent)
552{
553 pid->p_gain = div_fp(percent, 100);
554}
555
556static inline void pid_i_gain_set(struct _pid *pid, int percent)
557{
558 pid->i_gain = div_fp(percent, 100);
559}
560
561static inline void pid_d_gain_set(struct _pid *pid, int percent)
562{
563 pid->d_gain = div_fp(percent, 100);
564}
565
566static signed int pid_calc(struct _pid *pid, int32_t busy) 508static signed int pid_calc(struct _pid *pid, int32_t busy)
567{ 509{
568 signed int result; 510 signed int result;
@@ -600,23 +542,17 @@ static signed int pid_calc(struct _pid *pid, int32_t busy)
600 return (signed int)fp_toint(result); 542 return (signed int)fp_toint(result);
601} 543}
602 544
603static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu) 545static inline void intel_pstate_pid_reset(struct cpudata *cpu)
604{
605 pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
606 pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
607 pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
608
609 pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0);
610}
611
612static inline void intel_pstate_reset_all_pid(void)
613{ 546{
614 unsigned int cpu; 547 struct _pid *pid = &cpu->pid;
615 548
616 for_each_online_cpu(cpu) { 549 pid->p_gain = percent_fp(pid_params.p_gain_pct);
617 if (all_cpu_data[cpu]) 550 pid->d_gain = percent_fp(pid_params.d_gain_pct);
618 intel_pstate_busy_pid_reset(all_cpu_data[cpu]); 551 pid->i_gain = percent_fp(pid_params.i_gain_pct);
619 } 552 pid->setpoint = int_tofp(pid_params.setpoint);
553 pid->last_err = pid->setpoint - int_tofp(100);
554 pid->deadband = int_tofp(pid_params.deadband);
555 pid->integral = 0;
620} 556}
621 557
622static inline void update_turbo_state(void) 558static inline void update_turbo_state(void)
@@ -626,11 +562,19 @@ static inline void update_turbo_state(void)
626 562
627 cpu = all_cpu_data[0]; 563 cpu = all_cpu_data[0];
628 rdmsrl(MSR_IA32_MISC_ENABLE, misc_en); 564 rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
629 limits->turbo_disabled = 565 global.turbo_disabled =
630 (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE || 566 (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
631 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate); 567 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
632} 568}
633 569
570static int min_perf_pct_min(void)
571{
572 struct cpudata *cpu = all_cpu_data[0];
573
574 return DIV_ROUND_UP(cpu->pstate.min_pstate * 100,
575 cpu->pstate.turbo_pstate);
576}
577
634static s16 intel_pstate_get_epb(struct cpudata *cpu_data) 578static s16 intel_pstate_get_epb(struct cpudata *cpu_data)
635{ 579{
636 u64 epb; 580 u64 epb;
@@ -848,93 +792,80 @@ static struct freq_attr *hwp_cpufreq_attrs[] = {
848 NULL, 792 NULL,
849}; 793};
850 794
851static void intel_pstate_hwp_set(struct cpufreq_policy *policy) 795static void intel_pstate_hwp_set(unsigned int cpu)
852{ 796{
853 int min, hw_min, max, hw_max, cpu; 797 struct cpudata *cpu_data = all_cpu_data[cpu];
854 struct perf_limits *perf_limits = limits; 798 int min, hw_min, max, hw_max;
855 u64 value, cap; 799 u64 value, cap;
800 s16 epp;
856 801
857 for_each_cpu(cpu, policy->cpus) { 802 rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
858 struct cpudata *cpu_data = all_cpu_data[cpu]; 803 hw_min = HWP_LOWEST_PERF(cap);
859 s16 epp; 804 if (global.no_turbo)
860 805 hw_max = HWP_GUARANTEED_PERF(cap);
861 if (per_cpu_limits) 806 else
862 perf_limits = all_cpu_data[cpu]->perf_limits; 807 hw_max = HWP_HIGHEST_PERF(cap);
863
864 rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
865 hw_min = HWP_LOWEST_PERF(cap);
866 if (limits->no_turbo)
867 hw_max = HWP_GUARANTEED_PERF(cap);
868 else
869 hw_max = HWP_HIGHEST_PERF(cap);
870
871 min = fp_ext_toint(hw_max * perf_limits->min_perf);
872 808
873 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); 809 max = fp_ext_toint(hw_max * cpu_data->max_perf);
810 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
811 min = max;
812 else
813 min = fp_ext_toint(hw_max * cpu_data->min_perf);
874 814
875 value &= ~HWP_MIN_PERF(~0L); 815 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
876 value |= HWP_MIN_PERF(min);
877 816
878 max = fp_ext_toint(hw_max * perf_limits->max_perf); 817 value &= ~HWP_MIN_PERF(~0L);
879 value &= ~HWP_MAX_PERF(~0L); 818 value |= HWP_MIN_PERF(min);
880 value |= HWP_MAX_PERF(max);
881 819
882 if (cpu_data->epp_policy == cpu_data->policy) 820 value &= ~HWP_MAX_PERF(~0L);
883 goto skip_epp; 821 value |= HWP_MAX_PERF(max);
884 822
885 cpu_data->epp_policy = cpu_data->policy; 823 if (cpu_data->epp_policy == cpu_data->policy)
824 goto skip_epp;
886 825
887 if (cpu_data->epp_saved >= 0) { 826 cpu_data->epp_policy = cpu_data->policy;
888 epp = cpu_data->epp_saved;
889 cpu_data->epp_saved = -EINVAL;
890 goto update_epp;
891 }
892 827
893 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) { 828 if (cpu_data->epp_saved >= 0) {
894 epp = intel_pstate_get_epp(cpu_data, value); 829 epp = cpu_data->epp_saved;
895 cpu_data->epp_powersave = epp; 830 cpu_data->epp_saved = -EINVAL;
896 /* If EPP read was failed, then don't try to write */ 831 goto update_epp;
897 if (epp < 0) 832 }
898 goto skip_epp;
899 833
834 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
835 epp = intel_pstate_get_epp(cpu_data, value);
836 cpu_data->epp_powersave = epp;
837 /* If EPP read was failed, then don't try to write */
838 if (epp < 0)
839 goto skip_epp;
900 840
901 epp = 0; 841 epp = 0;
902 } else { 842 } else {
903 /* skip setting EPP, when saved value is invalid */ 843 /* skip setting EPP, when saved value is invalid */
904 if (cpu_data->epp_powersave < 0) 844 if (cpu_data->epp_powersave < 0)
905 goto skip_epp; 845 goto skip_epp;
906 846
907 /* 847 /*
908 * No need to restore EPP when it is not zero. This 848 * No need to restore EPP when it is not zero. This
909 * means: 849 * means:
910 * - Policy is not changed 850 * - Policy is not changed
911 * - user has manually changed 851 * - user has manually changed
912 * - Error reading EPB 852 * - Error reading EPB
913 */ 853 */
914 epp = intel_pstate_get_epp(cpu_data, value); 854 epp = intel_pstate_get_epp(cpu_data, value);
915 if (epp) 855 if (epp)
916 goto skip_epp; 856 goto skip_epp;
917 857
918 epp = cpu_data->epp_powersave; 858 epp = cpu_data->epp_powersave;
919 } 859 }
920update_epp: 860update_epp:
921 if (static_cpu_has(X86_FEATURE_HWP_EPP)) { 861 if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
922 value &= ~GENMASK_ULL(31, 24); 862 value &= ~GENMASK_ULL(31, 24);
923 value |= (u64)epp << 24; 863 value |= (u64)epp << 24;
924 } else { 864 } else {
925 intel_pstate_set_epb(cpu, epp); 865 intel_pstate_set_epb(cpu, epp);
926 }
927skip_epp:
928 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
929 } 866 }
930} 867skip_epp:
931 868 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
932static int intel_pstate_hwp_set_policy(struct cpufreq_policy *policy)
933{
934 if (hwp_active)
935 intel_pstate_hwp_set(policy);
936
937 return 0;
938} 869}
939 870
940static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy) 871static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
@@ -951,45 +882,38 @@ static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
951 882
952static int intel_pstate_resume(struct cpufreq_policy *policy) 883static int intel_pstate_resume(struct cpufreq_policy *policy)
953{ 884{
954 int ret;
955
956 if (!hwp_active) 885 if (!hwp_active)
957 return 0; 886 return 0;
958 887
959 mutex_lock(&intel_pstate_limits_lock); 888 mutex_lock(&intel_pstate_limits_lock);
960 889
961 all_cpu_data[policy->cpu]->epp_policy = 0; 890 all_cpu_data[policy->cpu]->epp_policy = 0;
962 891 intel_pstate_hwp_set(policy->cpu);
963 ret = intel_pstate_hwp_set_policy(policy);
964 892
965 mutex_unlock(&intel_pstate_limits_lock); 893 mutex_unlock(&intel_pstate_limits_lock);
966 894
967 return ret; 895 return 0;
968} 896}
969 897
970static void intel_pstate_update_policies(void) 898static void intel_pstate_update_policies(void)
971 __releases(&intel_pstate_limits_lock)
972 __acquires(&intel_pstate_limits_lock)
973{ 899{
974 struct perf_limits *saved_limits = limits;
975 int cpu; 900 int cpu;
976 901
977 mutex_unlock(&intel_pstate_limits_lock);
978
979 for_each_possible_cpu(cpu) 902 for_each_possible_cpu(cpu)
980 cpufreq_update_policy(cpu); 903 cpufreq_update_policy(cpu);
981
982 mutex_lock(&intel_pstate_limits_lock);
983
984 limits = saved_limits;
985} 904}
986 905
987/************************** debugfs begin ************************/ 906/************************** debugfs begin ************************/
988static int pid_param_set(void *data, u64 val) 907static int pid_param_set(void *data, u64 val)
989{ 908{
909 unsigned int cpu;
910
990 *(u32 *)data = val; 911 *(u32 *)data = val;
991 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; 912 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
992 intel_pstate_reset_all_pid(); 913 for_each_possible_cpu(cpu)
914 if (all_cpu_data[cpu])
915 intel_pstate_pid_reset(all_cpu_data[cpu]);
916
993 return 0; 917 return 0;
994} 918}
995 919
@@ -1060,7 +984,7 @@ static void intel_pstate_debug_hide_params(void)
1060 static ssize_t show_##file_name \ 984 static ssize_t show_##file_name \
1061 (struct kobject *kobj, struct attribute *attr, char *buf) \ 985 (struct kobject *kobj, struct attribute *attr, char *buf) \
1062 { \ 986 { \
1063 return sprintf(buf, "%u\n", limits->object); \ 987 return sprintf(buf, "%u\n", global.object); \
1064 } 988 }
1065 989
1066static ssize_t intel_pstate_show_status(char *buf); 990static ssize_t intel_pstate_show_status(char *buf);
@@ -1100,7 +1024,7 @@ static ssize_t show_turbo_pct(struct kobject *kobj,
1100 1024
1101 mutex_lock(&intel_pstate_driver_lock); 1025 mutex_lock(&intel_pstate_driver_lock);
1102 1026
1103 if (!driver_registered) { 1027 if (!intel_pstate_driver) {
1104 mutex_unlock(&intel_pstate_driver_lock); 1028 mutex_unlock(&intel_pstate_driver_lock);
1105 return -EAGAIN; 1029 return -EAGAIN;
1106 } 1030 }
@@ -1125,7 +1049,7 @@ static ssize_t show_num_pstates(struct kobject *kobj,
1125 1049
1126 mutex_lock(&intel_pstate_driver_lock); 1050 mutex_lock(&intel_pstate_driver_lock);
1127 1051
1128 if (!driver_registered) { 1052 if (!intel_pstate_driver) {
1129 mutex_unlock(&intel_pstate_driver_lock); 1053 mutex_unlock(&intel_pstate_driver_lock);
1130 return -EAGAIN; 1054 return -EAGAIN;
1131 } 1055 }
@@ -1145,16 +1069,16 @@ static ssize_t show_no_turbo(struct kobject *kobj,
1145 1069
1146 mutex_lock(&intel_pstate_driver_lock); 1070 mutex_lock(&intel_pstate_driver_lock);
1147 1071
1148 if (!driver_registered) { 1072 if (!intel_pstate_driver) {
1149 mutex_unlock(&intel_pstate_driver_lock); 1073 mutex_unlock(&intel_pstate_driver_lock);
1150 return -EAGAIN; 1074 return -EAGAIN;
1151 } 1075 }
1152 1076
1153 update_turbo_state(); 1077 update_turbo_state();
1154 if (limits->turbo_disabled) 1078 if (global.turbo_disabled)
1155 ret = sprintf(buf, "%u\n", limits->turbo_disabled); 1079 ret = sprintf(buf, "%u\n", global.turbo_disabled);
1156 else 1080 else
1157 ret = sprintf(buf, "%u\n", limits->no_turbo); 1081 ret = sprintf(buf, "%u\n", global.no_turbo);
1158 1082
1159 mutex_unlock(&intel_pstate_driver_lock); 1083 mutex_unlock(&intel_pstate_driver_lock);
1160 1084
@@ -1173,7 +1097,7 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
1173 1097
1174 mutex_lock(&intel_pstate_driver_lock); 1098 mutex_lock(&intel_pstate_driver_lock);
1175 1099
1176 if (!driver_registered) { 1100 if (!intel_pstate_driver) {
1177 mutex_unlock(&intel_pstate_driver_lock); 1101 mutex_unlock(&intel_pstate_driver_lock);
1178 return -EAGAIN; 1102 return -EAGAIN;
1179 } 1103 }
@@ -1181,19 +1105,28 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
1181 mutex_lock(&intel_pstate_limits_lock); 1105 mutex_lock(&intel_pstate_limits_lock);
1182 1106
1183 update_turbo_state(); 1107 update_turbo_state();
1184 if (limits->turbo_disabled) { 1108 if (global.turbo_disabled) {
1185 pr_warn("Turbo disabled by BIOS or unavailable on processor\n"); 1109 pr_warn("Turbo disabled by BIOS or unavailable on processor\n");
1186 mutex_unlock(&intel_pstate_limits_lock); 1110 mutex_unlock(&intel_pstate_limits_lock);
1187 mutex_unlock(&intel_pstate_driver_lock); 1111 mutex_unlock(&intel_pstate_driver_lock);
1188 return -EPERM; 1112 return -EPERM;
1189 } 1113 }
1190 1114
1191 limits->no_turbo = clamp_t(int, input, 0, 1); 1115 global.no_turbo = clamp_t(int, input, 0, 1);
1192 1116
1193 intel_pstate_update_policies(); 1117 if (global.no_turbo) {
1118 struct cpudata *cpu = all_cpu_data[0];
1119 int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate;
1120
1121 /* Squash the global minimum into the permitted range. */
1122 if (global.min_perf_pct > pct)
1123 global.min_perf_pct = pct;
1124 }
1194 1125
1195 mutex_unlock(&intel_pstate_limits_lock); 1126 mutex_unlock(&intel_pstate_limits_lock);
1196 1127
1128 intel_pstate_update_policies();
1129
1197 mutex_unlock(&intel_pstate_driver_lock); 1130 mutex_unlock(&intel_pstate_driver_lock);
1198 1131
1199 return count; 1132 return count;
@@ -1211,26 +1144,19 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
1211 1144
1212 mutex_lock(&intel_pstate_driver_lock); 1145 mutex_lock(&intel_pstate_driver_lock);
1213 1146
1214 if (!driver_registered) { 1147 if (!intel_pstate_driver) {
1215 mutex_unlock(&intel_pstate_driver_lock); 1148 mutex_unlock(&intel_pstate_driver_lock);
1216 return -EAGAIN; 1149 return -EAGAIN;
1217 } 1150 }
1218 1151
1219 mutex_lock(&intel_pstate_limits_lock); 1152 mutex_lock(&intel_pstate_limits_lock);
1220 1153
1221 limits->max_sysfs_pct = clamp_t(int, input, 0 , 100); 1154 global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100);
1222 limits->max_perf_pct = min(limits->max_policy_pct,
1223 limits->max_sysfs_pct);
1224 limits->max_perf_pct = max(limits->min_policy_pct,
1225 limits->max_perf_pct);
1226 limits->max_perf_pct = max(limits->min_perf_pct,
1227 limits->max_perf_pct);
1228 limits->max_perf = percent_ext_fp(limits->max_perf_pct);
1229
1230 intel_pstate_update_policies();
1231 1155
1232 mutex_unlock(&intel_pstate_limits_lock); 1156 mutex_unlock(&intel_pstate_limits_lock);
1233 1157
1158 intel_pstate_update_policies();
1159
1234 mutex_unlock(&intel_pstate_driver_lock); 1160 mutex_unlock(&intel_pstate_driver_lock);
1235 1161
1236 return count; 1162 return count;
@@ -1248,26 +1174,20 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
1248 1174
1249 mutex_lock(&intel_pstate_driver_lock); 1175 mutex_lock(&intel_pstate_driver_lock);
1250 1176
1251 if (!driver_registered) { 1177 if (!intel_pstate_driver) {
1252 mutex_unlock(&intel_pstate_driver_lock); 1178 mutex_unlock(&intel_pstate_driver_lock);
1253 return -EAGAIN; 1179 return -EAGAIN;
1254 } 1180 }
1255 1181
1256 mutex_lock(&intel_pstate_limits_lock); 1182 mutex_lock(&intel_pstate_limits_lock);
1257 1183
1258 limits->min_sysfs_pct = clamp_t(int, input, 0 , 100); 1184 global.min_perf_pct = clamp_t(int, input,
1259 limits->min_perf_pct = max(limits->min_policy_pct, 1185 min_perf_pct_min(), global.max_perf_pct);
1260 limits->min_sysfs_pct);
1261 limits->min_perf_pct = min(limits->max_policy_pct,
1262 limits->min_perf_pct);
1263 limits->min_perf_pct = min(limits->max_perf_pct,
1264 limits->min_perf_pct);
1265 limits->min_perf = percent_ext_fp(limits->min_perf_pct);
1266
1267 intel_pstate_update_policies();
1268 1186
1269 mutex_unlock(&intel_pstate_limits_lock); 1187 mutex_unlock(&intel_pstate_limits_lock);
1270 1188
1189 intel_pstate_update_policies();
1190
1271 mutex_unlock(&intel_pstate_driver_lock); 1191 mutex_unlock(&intel_pstate_driver_lock);
1272 1192
1273 return count; 1193 return count;
@@ -1387,7 +1307,7 @@ static u64 atom_get_val(struct cpudata *cpudata, int pstate)
1387 u32 vid; 1307 u32 vid;
1388 1308
1389 val = (u64)pstate << 8; 1309 val = (u64)pstate << 8;
1390 if (limits->no_turbo && !limits->turbo_disabled) 1310 if (global.no_turbo && !global.turbo_disabled)
1391 val |= (u64)1 << 32; 1311 val |= (u64)1 << 32;
1392 1312
1393 vid_fp = cpudata->vid.min + mul_fp( 1313 vid_fp = cpudata->vid.min + mul_fp(
@@ -1557,7 +1477,7 @@ static u64 core_get_val(struct cpudata *cpudata, int pstate)
1557 u64 val; 1477 u64 val;
1558 1478
1559 val = (u64)pstate << 8; 1479 val = (u64)pstate << 8;
1560 if (limits->no_turbo && !limits->turbo_disabled) 1480 if (global.no_turbo && !global.turbo_disabled)
1561 val |= (u64)1 << 32; 1481 val |= (u64)1 << 32;
1562 1482
1563 return val; 1483 return val;
@@ -1576,132 +1496,10 @@ static int knl_get_turbo_pstate(void)
1576 return ret; 1496 return ret;
1577} 1497}
1578 1498
1579static struct cpu_defaults core_params = { 1499static int intel_pstate_get_base_pstate(struct cpudata *cpu)
1580 .pid_policy = {
1581 .sample_rate_ms = 10,
1582 .deadband = 0,
1583 .setpoint = 97,
1584 .p_gain_pct = 20,
1585 .d_gain_pct = 0,
1586 .i_gain_pct = 0,
1587 },
1588 .funcs = {
1589 .get_max = core_get_max_pstate,
1590 .get_max_physical = core_get_max_pstate_physical,
1591 .get_min = core_get_min_pstate,
1592 .get_turbo = core_get_turbo_pstate,
1593 .get_scaling = core_get_scaling,
1594 .get_val = core_get_val,
1595 .get_target_pstate = get_target_pstate_use_performance,
1596 },
1597};
1598
1599static const struct cpu_defaults silvermont_params = {
1600 .pid_policy = {
1601 .sample_rate_ms = 10,
1602 .deadband = 0,
1603 .setpoint = 60,
1604 .p_gain_pct = 14,
1605 .d_gain_pct = 0,
1606 .i_gain_pct = 4,
1607 },
1608 .funcs = {
1609 .get_max = atom_get_max_pstate,
1610 .get_max_physical = atom_get_max_pstate,
1611 .get_min = atom_get_min_pstate,
1612 .get_turbo = atom_get_turbo_pstate,
1613 .get_val = atom_get_val,
1614 .get_scaling = silvermont_get_scaling,
1615 .get_vid = atom_get_vid,
1616 .get_target_pstate = get_target_pstate_use_cpu_load,
1617 },
1618};
1619
1620static const struct cpu_defaults airmont_params = {
1621 .pid_policy = {
1622 .sample_rate_ms = 10,
1623 .deadband = 0,
1624 .setpoint = 60,
1625 .p_gain_pct = 14,
1626 .d_gain_pct = 0,
1627 .i_gain_pct = 4,
1628 },
1629 .funcs = {
1630 .get_max = atom_get_max_pstate,
1631 .get_max_physical = atom_get_max_pstate,
1632 .get_min = atom_get_min_pstate,
1633 .get_turbo = atom_get_turbo_pstate,
1634 .get_val = atom_get_val,
1635 .get_scaling = airmont_get_scaling,
1636 .get_vid = atom_get_vid,
1637 .get_target_pstate = get_target_pstate_use_cpu_load,
1638 },
1639};
1640
1641static const struct cpu_defaults knl_params = {
1642 .pid_policy = {
1643 .sample_rate_ms = 10,
1644 .deadband = 0,
1645 .setpoint = 97,
1646 .p_gain_pct = 20,
1647 .d_gain_pct = 0,
1648 .i_gain_pct = 0,
1649 },
1650 .funcs = {
1651 .get_max = core_get_max_pstate,
1652 .get_max_physical = core_get_max_pstate_physical,
1653 .get_min = core_get_min_pstate,
1654 .get_turbo = knl_get_turbo_pstate,
1655 .get_scaling = core_get_scaling,
1656 .get_val = core_get_val,
1657 .get_target_pstate = get_target_pstate_use_performance,
1658 },
1659};
1660
1661static const struct cpu_defaults bxt_params = {
1662 .pid_policy = {
1663 .sample_rate_ms = 10,
1664 .deadband = 0,
1665 .setpoint = 60,
1666 .p_gain_pct = 14,
1667 .d_gain_pct = 0,
1668 .i_gain_pct = 4,
1669 },
1670 .funcs = {
1671 .get_max = core_get_max_pstate,
1672 .get_max_physical = core_get_max_pstate_physical,
1673 .get_min = core_get_min_pstate,
1674 .get_turbo = core_get_turbo_pstate,
1675 .get_scaling = core_get_scaling,
1676 .get_val = core_get_val,
1677 .get_target_pstate = get_target_pstate_use_cpu_load,
1678 },
1679};
1680
1681static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
1682{ 1500{
1683 int max_perf = cpu->pstate.turbo_pstate; 1501 return global.no_turbo || global.turbo_disabled ?
1684 int max_perf_adj; 1502 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1685 int min_perf;
1686 struct perf_limits *perf_limits = limits;
1687
1688 if (limits->no_turbo || limits->turbo_disabled)
1689 max_perf = cpu->pstate.max_pstate;
1690
1691 if (per_cpu_limits)
1692 perf_limits = cpu->perf_limits;
1693
1694 /*
1695 * performance can be limited by user through sysfs, by cpufreq
1696 * policy, or by cpu specific default values determined through
1697 * experimentation.
1698 */
1699 max_perf_adj = fp_ext_toint(max_perf * perf_limits->max_perf);
1700 *max = clamp_t(int, max_perf_adj,
1701 cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
1702
1703 min_perf = fp_ext_toint(max_perf * perf_limits->min_perf);
1704 *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
1705} 1503}
1706 1504
1707static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) 1505static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
@@ -1724,11 +1522,13 @@ static void intel_pstate_set_min_pstate(struct cpudata *cpu)
1724 1522
1725static void intel_pstate_max_within_limits(struct cpudata *cpu) 1523static void intel_pstate_max_within_limits(struct cpudata *cpu)
1726{ 1524{
1727 int min_pstate, max_pstate; 1525 int pstate;
1728 1526
1729 update_turbo_state(); 1527 update_turbo_state();
1730 intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate); 1528 pstate = intel_pstate_get_base_pstate(cpu);
1731 intel_pstate_set_pstate(cpu, max_pstate); 1529 pstate = max(cpu->pstate.min_pstate,
1530 fp_ext_toint(pstate * cpu->max_perf));
1531 intel_pstate_set_pstate(cpu, pstate);
1732} 1532}
1733 1533
1734static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) 1534static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
@@ -1789,7 +1589,11 @@ static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
1789 * that sample.time will always be reset before setting the utilization 1589 * that sample.time will always be reset before setting the utilization
1790 * update hook and make the caller skip the sample then. 1590 * update hook and make the caller skip the sample then.
1791 */ 1591 */
1792 return !!cpu->last_sample_time; 1592 if (cpu->last_sample_time) {
1593 intel_pstate_calc_avg_perf(cpu);
1594 return true;
1595 }
1596 return false;
1793} 1597}
1794 1598
1795static inline int32_t get_avg_frequency(struct cpudata *cpu) 1599static inline int32_t get_avg_frequency(struct cpudata *cpu)
@@ -1810,6 +1614,9 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
1810 int32_t busy_frac, boost; 1614 int32_t busy_frac, boost;
1811 int target, avg_pstate; 1615 int target, avg_pstate;
1812 1616
1617 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE)
1618 return cpu->pstate.turbo_pstate;
1619
1813 busy_frac = div_fp(sample->mperf, sample->tsc); 1620 busy_frac = div_fp(sample->mperf, sample->tsc);
1814 1621
1815 boost = cpu->iowait_boost; 1622 boost = cpu->iowait_boost;
@@ -1820,7 +1627,7 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
1820 1627
1821 sample->busy_scaled = busy_frac * 100; 1628 sample->busy_scaled = busy_frac * 100;
1822 1629
1823 target = limits->no_turbo || limits->turbo_disabled ? 1630 target = global.no_turbo || global.turbo_disabled ?
1824 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate; 1631 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1825 target += target >> 2; 1632 target += target >> 2;
1826 target = mul_fp(target, busy_frac); 1633 target = mul_fp(target, busy_frac);
@@ -1846,6 +1653,9 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
1846 int32_t perf_scaled, max_pstate, current_pstate, sample_ratio; 1653 int32_t perf_scaled, max_pstate, current_pstate, sample_ratio;
1847 u64 duration_ns; 1654 u64 duration_ns;
1848 1655
1656 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE)
1657 return cpu->pstate.turbo_pstate;
1658
1849 /* 1659 /*
1850 * perf_scaled is the ratio of the average P-state during the last 1660 * perf_scaled is the ratio of the average P-state during the last
1851 * sampling period to the P-state requested last time (in percent). 1661 * sampling period to the P-state requested last time (in percent).
@@ -1880,11 +1690,13 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
1880 1690
1881static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate) 1691static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
1882{ 1692{
1883 int max_perf, min_perf; 1693 int max_pstate = intel_pstate_get_base_pstate(cpu);
1694 int min_pstate;
1884 1695
1885 intel_pstate_get_min_max(cpu, &min_perf, &max_perf); 1696 min_pstate = max(cpu->pstate.min_pstate,
1886 pstate = clamp_t(int, pstate, min_perf, max_perf); 1697 fp_ext_toint(max_pstate * cpu->min_perf));
1887 return pstate; 1698 max_pstate = max(min_pstate, fp_ext_toint(max_pstate * cpu->max_perf));
1699 return clamp_t(int, pstate, min_pstate, max_pstate);
1888} 1700}
1889 1701
1890static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) 1702static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
@@ -1896,16 +1708,11 @@ static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
1896 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); 1708 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
1897} 1709}
1898 1710
1899static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) 1711static void intel_pstate_adjust_pstate(struct cpudata *cpu, int target_pstate)
1900{ 1712{
1901 int from, target_pstate; 1713 int from = cpu->pstate.current_pstate;
1902 struct sample *sample; 1714 struct sample *sample;
1903 1715
1904 from = cpu->pstate.current_pstate;
1905
1906 target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ?
1907 cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu);
1908
1909 update_turbo_state(); 1716 update_turbo_state();
1910 1717
1911 target_pstate = intel_pstate_prepare_request(cpu, target_pstate); 1718 target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
@@ -1924,76 +1731,155 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
1924 fp_toint(cpu->iowait_boost * 100)); 1731 fp_toint(cpu->iowait_boost * 100));
1925} 1732}
1926 1733
1734static void intel_pstate_update_util_hwp(struct update_util_data *data,
1735 u64 time, unsigned int flags)
1736{
1737 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1738 u64 delta_ns = time - cpu->sample.time;
1739
1740 if ((s64)delta_ns >= INTEL_PSTATE_HWP_SAMPLING_INTERVAL)
1741 intel_pstate_sample(cpu, time);
1742}
1743
1744static void intel_pstate_update_util_pid(struct update_util_data *data,
1745 u64 time, unsigned int flags)
1746{
1747 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1748 u64 delta_ns = time - cpu->sample.time;
1749
1750 if ((s64)delta_ns < pid_params.sample_rate_ns)
1751 return;
1752
1753 if (intel_pstate_sample(cpu, time)) {
1754 int target_pstate;
1755
1756 target_pstate = get_target_pstate_use_performance(cpu);
1757 intel_pstate_adjust_pstate(cpu, target_pstate);
1758 }
1759}
1760
1927static void intel_pstate_update_util(struct update_util_data *data, u64 time, 1761static void intel_pstate_update_util(struct update_util_data *data, u64 time,
1928 unsigned int flags) 1762 unsigned int flags)
1929{ 1763{
1930 struct cpudata *cpu = container_of(data, struct cpudata, update_util); 1764 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1931 u64 delta_ns; 1765 u64 delta_ns;
1932 1766
1933 if (pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load) { 1767 if (flags & SCHED_CPUFREQ_IOWAIT) {
1934 if (flags & SCHED_CPUFREQ_IOWAIT) { 1768 cpu->iowait_boost = int_tofp(1);
1935 cpu->iowait_boost = int_tofp(1); 1769 } else if (cpu->iowait_boost) {
1936 } else if (cpu->iowait_boost) { 1770 /* Clear iowait_boost if the CPU may have been idle. */
1937 /* Clear iowait_boost if the CPU may have been idle. */ 1771 delta_ns = time - cpu->last_update;
1938 delta_ns = time - cpu->last_update; 1772 if (delta_ns > TICK_NSEC)
1939 if (delta_ns > TICK_NSEC) 1773 cpu->iowait_boost = 0;
1940 cpu->iowait_boost = 0;
1941 }
1942 cpu->last_update = time;
1943 } 1774 }
1944 1775 cpu->last_update = time;
1945 delta_ns = time - cpu->sample.time; 1776 delta_ns = time - cpu->sample.time;
1946 if ((s64)delta_ns >= pid_params.sample_rate_ns) { 1777 if ((s64)delta_ns < INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL)
1947 bool sample_taken = intel_pstate_sample(cpu, time); 1778 return;
1948 1779
1949 if (sample_taken) { 1780 if (intel_pstate_sample(cpu, time)) {
1950 intel_pstate_calc_avg_perf(cpu); 1781 int target_pstate;
1951 if (!hwp_active) 1782
1952 intel_pstate_adjust_busy_pstate(cpu); 1783 target_pstate = get_target_pstate_use_cpu_load(cpu);
1953 } 1784 intel_pstate_adjust_pstate(cpu, target_pstate);
1954 } 1785 }
1955} 1786}
1956 1787
1788static struct pstate_funcs core_funcs = {
1789 .get_max = core_get_max_pstate,
1790 .get_max_physical = core_get_max_pstate_physical,
1791 .get_min = core_get_min_pstate,
1792 .get_turbo = core_get_turbo_pstate,
1793 .get_scaling = core_get_scaling,
1794 .get_val = core_get_val,
1795 .update_util = intel_pstate_update_util_pid,
1796};
1797
1798static const struct pstate_funcs silvermont_funcs = {
1799 .get_max = atom_get_max_pstate,
1800 .get_max_physical = atom_get_max_pstate,
1801 .get_min = atom_get_min_pstate,
1802 .get_turbo = atom_get_turbo_pstate,
1803 .get_val = atom_get_val,
1804 .get_scaling = silvermont_get_scaling,
1805 .get_vid = atom_get_vid,
1806 .update_util = intel_pstate_update_util,
1807};
1808
1809static const struct pstate_funcs airmont_funcs = {
1810 .get_max = atom_get_max_pstate,
1811 .get_max_physical = atom_get_max_pstate,
1812 .get_min = atom_get_min_pstate,
1813 .get_turbo = atom_get_turbo_pstate,
1814 .get_val = atom_get_val,
1815 .get_scaling = airmont_get_scaling,
1816 .get_vid = atom_get_vid,
1817 .update_util = intel_pstate_update_util,
1818};
1819
1820static const struct pstate_funcs knl_funcs = {
1821 .get_max = core_get_max_pstate,
1822 .get_max_physical = core_get_max_pstate_physical,
1823 .get_min = core_get_min_pstate,
1824 .get_turbo = knl_get_turbo_pstate,
1825 .get_scaling = core_get_scaling,
1826 .get_val = core_get_val,
1827 .update_util = intel_pstate_update_util_pid,
1828};
1829
1830static const struct pstate_funcs bxt_funcs = {
1831 .get_max = core_get_max_pstate,
1832 .get_max_physical = core_get_max_pstate_physical,
1833 .get_min = core_get_min_pstate,
1834 .get_turbo = core_get_turbo_pstate,
1835 .get_scaling = core_get_scaling,
1836 .get_val = core_get_val,
1837 .update_util = intel_pstate_update_util,
1838};
1839
1957#define ICPU(model, policy) \ 1840#define ICPU(model, policy) \
1958 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\ 1841 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
1959 (unsigned long)&policy } 1842 (unsigned long)&policy }
1960 1843
1961static const struct x86_cpu_id intel_pstate_cpu_ids[] = { 1844static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1962 ICPU(INTEL_FAM6_SANDYBRIDGE, core_params), 1845 ICPU(INTEL_FAM6_SANDYBRIDGE, core_funcs),
1963 ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_params), 1846 ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_funcs),
1964 ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_params), 1847 ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_funcs),
1965 ICPU(INTEL_FAM6_IVYBRIDGE, core_params), 1848 ICPU(INTEL_FAM6_IVYBRIDGE, core_funcs),
1966 ICPU(INTEL_FAM6_HASWELL_CORE, core_params), 1849 ICPU(INTEL_FAM6_HASWELL_CORE, core_funcs),
1967 ICPU(INTEL_FAM6_BROADWELL_CORE, core_params), 1850 ICPU(INTEL_FAM6_BROADWELL_CORE, core_funcs),
1968 ICPU(INTEL_FAM6_IVYBRIDGE_X, core_params), 1851 ICPU(INTEL_FAM6_IVYBRIDGE_X, core_funcs),
1969 ICPU(INTEL_FAM6_HASWELL_X, core_params), 1852 ICPU(INTEL_FAM6_HASWELL_X, core_funcs),
1970 ICPU(INTEL_FAM6_HASWELL_ULT, core_params), 1853 ICPU(INTEL_FAM6_HASWELL_ULT, core_funcs),
1971 ICPU(INTEL_FAM6_HASWELL_GT3E, core_params), 1854 ICPU(INTEL_FAM6_HASWELL_GT3E, core_funcs),
1972 ICPU(INTEL_FAM6_BROADWELL_GT3E, core_params), 1855 ICPU(INTEL_FAM6_BROADWELL_GT3E, core_funcs),
1973 ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_params), 1856 ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_funcs),
1974 ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_params), 1857 ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_funcs),
1975 ICPU(INTEL_FAM6_BROADWELL_X, core_params), 1858 ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
1976 ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params), 1859 ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs),
1977 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), 1860 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
1978 ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params), 1861 ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_funcs),
1979 ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_params), 1862 ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_funcs),
1980 ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_params), 1863 ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_funcs),
1864 ICPU(INTEL_FAM6_ATOM_GEMINI_LAKE, bxt_funcs),
1981 {} 1865 {}
1982}; 1866};
1983MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); 1867MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
1984 1868
1985static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { 1869static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
1986 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), 1870 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
1987 ICPU(INTEL_FAM6_BROADWELL_X, core_params), 1871 ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
1988 ICPU(INTEL_FAM6_SKYLAKE_X, core_params), 1872 ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
1989 {} 1873 {}
1990}; 1874};
1991 1875
1992static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = { 1876static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
1993 ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_params), 1877 ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_funcs),
1994 {} 1878 {}
1995}; 1879};
1996 1880
1881static bool pid_in_use(void);
1882
1997static int intel_pstate_init_cpu(unsigned int cpunum) 1883static int intel_pstate_init_cpu(unsigned int cpunum)
1998{ 1884{
1999 struct cpudata *cpu; 1885 struct cpudata *cpu;
@@ -2001,18 +1887,11 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
2001 cpu = all_cpu_data[cpunum]; 1887 cpu = all_cpu_data[cpunum];
2002 1888
2003 if (!cpu) { 1889 if (!cpu) {
2004 unsigned int size = sizeof(struct cpudata); 1890 cpu = kzalloc(sizeof(*cpu), GFP_KERNEL);
2005
2006 if (per_cpu_limits)
2007 size += sizeof(struct perf_limits);
2008
2009 cpu = kzalloc(size, GFP_KERNEL);
2010 if (!cpu) 1891 if (!cpu)
2011 return -ENOMEM; 1892 return -ENOMEM;
2012 1893
2013 all_cpu_data[cpunum] = cpu; 1894 all_cpu_data[cpunum] = cpu;
2014 if (per_cpu_limits)
2015 cpu->perf_limits = (struct perf_limits *)(cpu + 1);
2016 1895
2017 cpu->epp_default = -EINVAL; 1896 cpu->epp_default = -EINVAL;
2018 cpu->epp_powersave = -EINVAL; 1897 cpu->epp_powersave = -EINVAL;
@@ -2031,14 +1910,12 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
2031 intel_pstate_disable_ee(cpunum); 1910 intel_pstate_disable_ee(cpunum);
2032 1911
2033 intel_pstate_hwp_enable(cpu); 1912 intel_pstate_hwp_enable(cpu);
2034 pid_params.sample_rate_ms = 50; 1913 } else if (pid_in_use()) {
2035 pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC; 1914 intel_pstate_pid_reset(cpu);
2036 } 1915 }
2037 1916
2038 intel_pstate_get_cpu_pstates(cpu); 1917 intel_pstate_get_cpu_pstates(cpu);
2039 1918
2040 intel_pstate_busy_pid_reset(cpu);
2041
2042 pr_debug("controlling: cpu %d\n", cpunum); 1919 pr_debug("controlling: cpu %d\n", cpunum);
2043 1920
2044 return 0; 1921 return 0;
@@ -2061,7 +1938,7 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
2061 /* Prevent intel_pstate_update_util() from using stale data. */ 1938 /* Prevent intel_pstate_update_util() from using stale data. */
2062 cpu->sample.time = 0; 1939 cpu->sample.time = 0;
2063 cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, 1940 cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
2064 intel_pstate_update_util); 1941 pstate_funcs.update_util);
2065 cpu->update_util_set = true; 1942 cpu->update_util_set = true;
2066} 1943}
2067 1944
@@ -2077,46 +1954,68 @@ static void intel_pstate_clear_update_util_hook(unsigned int cpu)
2077 synchronize_sched(); 1954 synchronize_sched();
2078} 1955}
2079 1956
1957static int intel_pstate_get_max_freq(struct cpudata *cpu)
1958{
1959 return global.turbo_disabled || global.no_turbo ?
1960 cpu->pstate.max_freq : cpu->pstate.turbo_freq;
1961}
1962
2080static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, 1963static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy,
2081 struct perf_limits *limits) 1964 struct cpudata *cpu)
2082{ 1965{
1966 int max_freq = intel_pstate_get_max_freq(cpu);
2083 int32_t max_policy_perf, min_policy_perf; 1967 int32_t max_policy_perf, min_policy_perf;
2084 1968
2085 max_policy_perf = div_ext_fp(policy->max, policy->cpuinfo.max_freq); 1969 max_policy_perf = div_ext_fp(policy->max, max_freq);
2086 max_policy_perf = clamp_t(int32_t, max_policy_perf, 0, int_ext_tofp(1)); 1970 max_policy_perf = clamp_t(int32_t, max_policy_perf, 0, int_ext_tofp(1));
2087 if (policy->max == policy->min) { 1971 if (policy->max == policy->min) {
2088 min_policy_perf = max_policy_perf; 1972 min_policy_perf = max_policy_perf;
2089 } else { 1973 } else {
2090 min_policy_perf = div_ext_fp(policy->min, 1974 min_policy_perf = div_ext_fp(policy->min, max_freq);
2091 policy->cpuinfo.max_freq);
2092 min_policy_perf = clamp_t(int32_t, min_policy_perf, 1975 min_policy_perf = clamp_t(int32_t, min_policy_perf,
2093 0, max_policy_perf); 1976 0, max_policy_perf);
2094 } 1977 }
2095 1978
2096 /* Normalize user input to [min_perf, max_perf] */ 1979 /* Normalize user input to [min_perf, max_perf] */
2097 limits->min_perf = max(min_policy_perf, 1980 if (per_cpu_limits) {
2098 percent_ext_fp(limits->min_sysfs_pct)); 1981 cpu->min_perf = min_policy_perf;
2099 limits->min_perf = min(limits->min_perf, max_policy_perf); 1982 cpu->max_perf = max_policy_perf;
2100 limits->max_perf = min(max_policy_perf, 1983 } else {
2101 percent_ext_fp(limits->max_sysfs_pct)); 1984 int32_t global_min, global_max;
2102 limits->max_perf = max(min_policy_perf, limits->max_perf); 1985
1986 /* Global limits are in percent of the maximum turbo P-state. */
1987 global_max = percent_ext_fp(global.max_perf_pct);
1988 global_min = percent_ext_fp(global.min_perf_pct);
1989 if (max_freq != cpu->pstate.turbo_freq) {
1990 int32_t turbo_factor;
1991
1992 turbo_factor = div_ext_fp(cpu->pstate.turbo_pstate,
1993 cpu->pstate.max_pstate);
1994 global_min = mul_ext_fp(global_min, turbo_factor);
1995 global_max = mul_ext_fp(global_max, turbo_factor);
1996 }
1997 global_min = clamp_t(int32_t, global_min, 0, global_max);
1998
1999 cpu->min_perf = max(min_policy_perf, global_min);
2000 cpu->min_perf = min(cpu->min_perf, max_policy_perf);
2001 cpu->max_perf = min(max_policy_perf, global_max);
2002 cpu->max_perf = max(min_policy_perf, cpu->max_perf);
2103 2003
2104 /* Make sure min_perf <= max_perf */ 2004 /* Make sure min_perf <= max_perf */
2105 limits->min_perf = min(limits->min_perf, limits->max_perf); 2005 cpu->min_perf = min(cpu->min_perf, cpu->max_perf);
2006 }
2106 2007
2107 limits->max_perf = round_up(limits->max_perf, EXT_FRAC_BITS); 2008 cpu->max_perf = round_up(cpu->max_perf, EXT_FRAC_BITS);
2108 limits->min_perf = round_up(limits->min_perf, EXT_FRAC_BITS); 2009 cpu->min_perf = round_up(cpu->min_perf, EXT_FRAC_BITS);
2109 limits->max_perf_pct = fp_ext_toint(limits->max_perf * 100);
2110 limits->min_perf_pct = fp_ext_toint(limits->min_perf * 100);
2111 2010
2112 pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu, 2011 pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu,
2113 limits->max_perf_pct, limits->min_perf_pct); 2012 fp_ext_toint(cpu->max_perf * 100),
2013 fp_ext_toint(cpu->min_perf * 100));
2114} 2014}
2115 2015
2116static int intel_pstate_set_policy(struct cpufreq_policy *policy) 2016static int intel_pstate_set_policy(struct cpufreq_policy *policy)
2117{ 2017{
2118 struct cpudata *cpu; 2018 struct cpudata *cpu;
2119 struct perf_limits *perf_limits = NULL;
2120 2019
2121 if (!policy->cpuinfo.max_freq) 2020 if (!policy->cpuinfo.max_freq)
2122 return -ENODEV; 2021 return -ENODEV;
@@ -2127,34 +2026,9 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
2127 cpu = all_cpu_data[policy->cpu]; 2026 cpu = all_cpu_data[policy->cpu];
2128 cpu->policy = policy->policy; 2027 cpu->policy = policy->policy;
2129 2028
2130 if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
2131 policy->max < policy->cpuinfo.max_freq &&
2132 policy->max > cpu->pstate.max_pstate * cpu->pstate.scaling) {
2133 pr_debug("policy->max > max non turbo frequency\n");
2134 policy->max = policy->cpuinfo.max_freq;
2135 }
2136
2137 if (per_cpu_limits)
2138 perf_limits = cpu->perf_limits;
2139
2140 mutex_lock(&intel_pstate_limits_lock); 2029 mutex_lock(&intel_pstate_limits_lock);
2141 2030
2142 if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) { 2031 intel_pstate_update_perf_limits(policy, cpu);
2143 pr_debug("set performance\n");
2144 if (!perf_limits) {
2145 limits = &performance_limits;
2146 perf_limits = limits;
2147 }
2148 } else {
2149 pr_debug("set powersave\n");
2150 if (!perf_limits) {
2151 limits = &powersave_limits;
2152 perf_limits = limits;
2153 }
2154
2155 }
2156
2157 intel_pstate_update_perf_limits(policy, perf_limits);
2158 2032
2159 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { 2033 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
2160 /* 2034 /*
@@ -2167,45 +2041,38 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
2167 2041
2168 intel_pstate_set_update_util_hook(policy->cpu); 2042 intel_pstate_set_update_util_hook(policy->cpu);
2169 2043
2170 intel_pstate_hwp_set_policy(policy); 2044 if (hwp_active)
2045 intel_pstate_hwp_set(policy->cpu);
2171 2046
2172 mutex_unlock(&intel_pstate_limits_lock); 2047 mutex_unlock(&intel_pstate_limits_lock);
2173 2048
2174 return 0; 2049 return 0;
2175} 2050}
2176 2051
2052static void intel_pstate_adjust_policy_max(struct cpufreq_policy *policy,
2053 struct cpudata *cpu)
2054{
2055 if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
2056 policy->max < policy->cpuinfo.max_freq &&
2057 policy->max > cpu->pstate.max_freq) {
2058 pr_debug("policy->max > max non turbo frequency\n");
2059 policy->max = policy->cpuinfo.max_freq;
2060 }
2061}
2062
2177static int intel_pstate_verify_policy(struct cpufreq_policy *policy) 2063static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
2178{ 2064{
2179 struct cpudata *cpu = all_cpu_data[policy->cpu]; 2065 struct cpudata *cpu = all_cpu_data[policy->cpu];
2180 struct perf_limits *perf_limits;
2181
2182 if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
2183 perf_limits = &performance_limits;
2184 else
2185 perf_limits = &powersave_limits;
2186 2066
2187 update_turbo_state(); 2067 update_turbo_state();
2188 policy->cpuinfo.max_freq = perf_limits->turbo_disabled || 2068 cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
2189 perf_limits->no_turbo ? 2069 intel_pstate_get_max_freq(cpu));
2190 cpu->pstate.max_freq :
2191 cpu->pstate.turbo_freq;
2192
2193 cpufreq_verify_within_cpu_limits(policy);
2194 2070
2195 if (policy->policy != CPUFREQ_POLICY_POWERSAVE && 2071 if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
2196 policy->policy != CPUFREQ_POLICY_PERFORMANCE) 2072 policy->policy != CPUFREQ_POLICY_PERFORMANCE)
2197 return -EINVAL; 2073 return -EINVAL;
2198 2074
2199 /* When per-CPU limits are used, sysfs limits are not used */ 2075 intel_pstate_adjust_policy_max(policy, cpu);
2200 if (!per_cpu_limits) {
2201 unsigned int max_freq, min_freq;
2202
2203 max_freq = policy->cpuinfo.max_freq *
2204 perf_limits->max_sysfs_pct / 100;
2205 min_freq = policy->cpuinfo.max_freq *
2206 perf_limits->min_sysfs_pct / 100;
2207 cpufreq_verify_within_limits(policy, min_freq, max_freq);
2208 }
2209 2076
2210 return 0; 2077 return 0;
2211} 2078}
@@ -2246,8 +2113,8 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
2246 2113
2247 cpu = all_cpu_data[policy->cpu]; 2114 cpu = all_cpu_data[policy->cpu];
2248 2115
2249 if (per_cpu_limits) 2116 cpu->max_perf = int_ext_tofp(1);
2250 intel_pstate_init_limits(cpu->perf_limits); 2117 cpu->min_perf = 0;
2251 2118
2252 policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling; 2119 policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
2253 policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling; 2120 policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
@@ -2255,7 +2122,7 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
2255 /* cpuinfo and default policy values */ 2122 /* cpuinfo and default policy values */
2256 policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling; 2123 policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
2257 update_turbo_state(); 2124 update_turbo_state();
2258 policy->cpuinfo.max_freq = limits->turbo_disabled ? 2125 policy->cpuinfo.max_freq = global.turbo_disabled ?
2259 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate; 2126 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
2260 policy->cpuinfo.max_freq *= cpu->pstate.scaling; 2127 policy->cpuinfo.max_freq *= cpu->pstate.scaling;
2261 2128
@@ -2275,7 +2142,7 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
2275 return ret; 2142 return ret;
2276 2143
2277 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; 2144 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
2278 if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100) 2145 if (IS_ENABLED(CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE))
2279 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 2146 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
2280 else 2147 else
2281 policy->policy = CPUFREQ_POLICY_POWERSAVE; 2148 policy->policy = CPUFREQ_POLICY_POWERSAVE;
@@ -2301,32 +2168,14 @@ static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
2301 struct cpudata *cpu = all_cpu_data[policy->cpu]; 2168 struct cpudata *cpu = all_cpu_data[policy->cpu];
2302 2169
2303 update_turbo_state(); 2170 update_turbo_state();
2304 policy->cpuinfo.max_freq = limits->turbo_disabled ? 2171 cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
2305 cpu->pstate.max_freq : cpu->pstate.turbo_freq; 2172 intel_pstate_get_max_freq(cpu));
2306
2307 cpufreq_verify_within_cpu_limits(policy);
2308 2173
2309 return 0; 2174 intel_pstate_adjust_policy_max(policy, cpu);
2310}
2311
2312static unsigned int intel_cpufreq_turbo_update(struct cpudata *cpu,
2313 struct cpufreq_policy *policy,
2314 unsigned int target_freq)
2315{
2316 unsigned int max_freq;
2317
2318 update_turbo_state();
2319
2320 max_freq = limits->no_turbo || limits->turbo_disabled ?
2321 cpu->pstate.max_freq : cpu->pstate.turbo_freq;
2322 policy->cpuinfo.max_freq = max_freq;
2323 if (policy->max > max_freq)
2324 policy->max = max_freq;
2325 2175
2326 if (target_freq > max_freq) 2176 intel_pstate_update_perf_limits(policy, cpu);
2327 target_freq = max_freq;
2328 2177
2329 return target_freq; 2178 return 0;
2330} 2179}
2331 2180
2332static int intel_cpufreq_target(struct cpufreq_policy *policy, 2181static int intel_cpufreq_target(struct cpufreq_policy *policy,
@@ -2337,8 +2186,10 @@ static int intel_cpufreq_target(struct cpufreq_policy *policy,
2337 struct cpufreq_freqs freqs; 2186 struct cpufreq_freqs freqs;
2338 int target_pstate; 2187 int target_pstate;
2339 2188
2189 update_turbo_state();
2190
2340 freqs.old = policy->cur; 2191 freqs.old = policy->cur;
2341 freqs.new = intel_cpufreq_turbo_update(cpu, policy, target_freq); 2192 freqs.new = target_freq;
2342 2193
2343 cpufreq_freq_transition_begin(policy, &freqs); 2194 cpufreq_freq_transition_begin(policy, &freqs);
2344 switch (relation) { 2195 switch (relation) {
@@ -2370,7 +2221,8 @@ static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
2370 struct cpudata *cpu = all_cpu_data[policy->cpu]; 2221 struct cpudata *cpu = all_cpu_data[policy->cpu];
2371 int target_pstate; 2222 int target_pstate;
2372 2223
2373 target_freq = intel_cpufreq_turbo_update(cpu, policy, target_freq); 2224 update_turbo_state();
2225
2374 target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling); 2226 target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
2375 target_pstate = intel_pstate_prepare_request(cpu, target_pstate); 2227 target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
2376 intel_pstate_update_pstate(cpu, target_pstate); 2228 intel_pstate_update_pstate(cpu, target_pstate);
@@ -2402,7 +2254,13 @@ static struct cpufreq_driver intel_cpufreq = {
2402 .name = "intel_cpufreq", 2254 .name = "intel_cpufreq",
2403}; 2255};
2404 2256
2405static struct cpufreq_driver *intel_pstate_driver = &intel_pstate; 2257static struct cpufreq_driver *default_driver = &intel_pstate;
2258
2259static bool pid_in_use(void)
2260{
2261 return intel_pstate_driver == &intel_pstate &&
2262 pstate_funcs.update_util == intel_pstate_update_util_pid;
2263}
2406 2264
2407static void intel_pstate_driver_cleanup(void) 2265static void intel_pstate_driver_cleanup(void)
2408{ 2266{
@@ -2419,32 +2277,26 @@ static void intel_pstate_driver_cleanup(void)
2419 } 2277 }
2420 } 2278 }
2421 put_online_cpus(); 2279 put_online_cpus();
2280 intel_pstate_driver = NULL;
2422} 2281}
2423 2282
2424static int intel_pstate_register_driver(void) 2283static int intel_pstate_register_driver(struct cpufreq_driver *driver)
2425{ 2284{
2426 int ret; 2285 int ret;
2427 2286
2428 intel_pstate_init_limits(&powersave_limits); 2287 memset(&global, 0, sizeof(global));
2429 intel_pstate_set_performance_limits(&performance_limits); 2288 global.max_perf_pct = 100;
2430 if (IS_ENABLED(CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE) &&
2431 intel_pstate_driver == &intel_pstate)
2432 limits = &performance_limits;
2433 else
2434 limits = &powersave_limits;
2435 2289
2290 intel_pstate_driver = driver;
2436 ret = cpufreq_register_driver(intel_pstate_driver); 2291 ret = cpufreq_register_driver(intel_pstate_driver);
2437 if (ret) { 2292 if (ret) {
2438 intel_pstate_driver_cleanup(); 2293 intel_pstate_driver_cleanup();
2439 return ret; 2294 return ret;
2440 } 2295 }
2441 2296
2442 mutex_lock(&intel_pstate_limits_lock); 2297 global.min_perf_pct = min_perf_pct_min();
2443 driver_registered = true;
2444 mutex_unlock(&intel_pstate_limits_lock);
2445 2298
2446 if (intel_pstate_driver == &intel_pstate && !hwp_active && 2299 if (pid_in_use())
2447 pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load)
2448 intel_pstate_debug_expose_params(); 2300 intel_pstate_debug_expose_params();
2449 2301
2450 return 0; 2302 return 0;
@@ -2455,14 +2307,9 @@ static int intel_pstate_unregister_driver(void)
2455 if (hwp_active) 2307 if (hwp_active)
2456 return -EBUSY; 2308 return -EBUSY;
2457 2309
2458 if (intel_pstate_driver == &intel_pstate && !hwp_active && 2310 if (pid_in_use())
2459 pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load)
2460 intel_pstate_debug_hide_params(); 2311 intel_pstate_debug_hide_params();
2461 2312
2462 mutex_lock(&intel_pstate_limits_lock);
2463 driver_registered = false;
2464 mutex_unlock(&intel_pstate_limits_lock);
2465
2466 cpufreq_unregister_driver(intel_pstate_driver); 2313 cpufreq_unregister_driver(intel_pstate_driver);
2467 intel_pstate_driver_cleanup(); 2314 intel_pstate_driver_cleanup();
2468 2315
@@ -2471,7 +2318,7 @@ static int intel_pstate_unregister_driver(void)
2471 2318
2472static ssize_t intel_pstate_show_status(char *buf) 2319static ssize_t intel_pstate_show_status(char *buf)
2473{ 2320{
2474 if (!driver_registered) 2321 if (!intel_pstate_driver)
2475 return sprintf(buf, "off\n"); 2322 return sprintf(buf, "off\n");
2476 2323
2477 return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ? 2324 return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ?
@@ -2483,11 +2330,11 @@ static int intel_pstate_update_status(const char *buf, size_t size)
2483 int ret; 2330 int ret;
2484 2331
2485 if (size == 3 && !strncmp(buf, "off", size)) 2332 if (size == 3 && !strncmp(buf, "off", size))
2486 return driver_registered ? 2333 return intel_pstate_driver ?
2487 intel_pstate_unregister_driver() : -EINVAL; 2334 intel_pstate_unregister_driver() : -EINVAL;
2488 2335
2489 if (size == 6 && !strncmp(buf, "active", size)) { 2336 if (size == 6 && !strncmp(buf, "active", size)) {
2490 if (driver_registered) { 2337 if (intel_pstate_driver) {
2491 if (intel_pstate_driver == &intel_pstate) 2338 if (intel_pstate_driver == &intel_pstate)
2492 return 0; 2339 return 0;
2493 2340
@@ -2496,13 +2343,12 @@ static int intel_pstate_update_status(const char *buf, size_t size)
2496 return ret; 2343 return ret;
2497 } 2344 }
2498 2345
2499 intel_pstate_driver = &intel_pstate; 2346 return intel_pstate_register_driver(&intel_pstate);
2500 return intel_pstate_register_driver();
2501 } 2347 }
2502 2348
2503 if (size == 7 && !strncmp(buf, "passive", size)) { 2349 if (size == 7 && !strncmp(buf, "passive", size)) {
2504 if (driver_registered) { 2350 if (intel_pstate_driver) {
2505 if (intel_pstate_driver != &intel_pstate) 2351 if (intel_pstate_driver == &intel_cpufreq)
2506 return 0; 2352 return 0;
2507 2353
2508 ret = intel_pstate_unregister_driver(); 2354 ret = intel_pstate_unregister_driver();
@@ -2510,8 +2356,7 @@ static int intel_pstate_update_status(const char *buf, size_t size)
2510 return ret; 2356 return ret;
2511 } 2357 }
2512 2358
2513 intel_pstate_driver = &intel_cpufreq; 2359 return intel_pstate_register_driver(&intel_cpufreq);
2514 return intel_pstate_register_driver();
2515 } 2360 }
2516 2361
2517 return -EINVAL; 2362 return -EINVAL;
@@ -2532,23 +2377,17 @@ static int __init intel_pstate_msrs_not_valid(void)
2532 return 0; 2377 return 0;
2533} 2378}
2534 2379
2535static void __init copy_pid_params(struct pstate_adjust_policy *policy)
2536{
2537 pid_params.sample_rate_ms = policy->sample_rate_ms;
2538 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
2539 pid_params.p_gain_pct = policy->p_gain_pct;
2540 pid_params.i_gain_pct = policy->i_gain_pct;
2541 pid_params.d_gain_pct = policy->d_gain_pct;
2542 pid_params.deadband = policy->deadband;
2543 pid_params.setpoint = policy->setpoint;
2544}
2545
2546#ifdef CONFIG_ACPI 2380#ifdef CONFIG_ACPI
2547static void intel_pstate_use_acpi_profile(void) 2381static void intel_pstate_use_acpi_profile(void)
2548{ 2382{
2549 if (acpi_gbl_FADT.preferred_profile == PM_MOBILE) 2383 switch (acpi_gbl_FADT.preferred_profile) {
2550 pstate_funcs.get_target_pstate = 2384 case PM_MOBILE:
2551 get_target_pstate_use_cpu_load; 2385 case PM_TABLET:
2386 case PM_APPLIANCE_PC:
2387 case PM_DESKTOP:
2388 case PM_WORKSTATION:
2389 pstate_funcs.update_util = intel_pstate_update_util;
2390 }
2552} 2391}
2553#else 2392#else
2554static void intel_pstate_use_acpi_profile(void) 2393static void intel_pstate_use_acpi_profile(void)
@@ -2565,7 +2404,7 @@ static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
2565 pstate_funcs.get_scaling = funcs->get_scaling; 2404 pstate_funcs.get_scaling = funcs->get_scaling;
2566 pstate_funcs.get_val = funcs->get_val; 2405 pstate_funcs.get_val = funcs->get_val;
2567 pstate_funcs.get_vid = funcs->get_vid; 2406 pstate_funcs.get_vid = funcs->get_vid;
2568 pstate_funcs.get_target_pstate = funcs->get_target_pstate; 2407 pstate_funcs.update_util = funcs->update_util;
2569 2408
2570 intel_pstate_use_acpi_profile(); 2409 intel_pstate_use_acpi_profile();
2571} 2410}
@@ -2704,28 +2543,30 @@ static const struct x86_cpu_id hwp_support_ids[] __initconst = {
2704 2543
2705static int __init intel_pstate_init(void) 2544static int __init intel_pstate_init(void)
2706{ 2545{
2707 const struct x86_cpu_id *id; 2546 int rc;
2708 struct cpu_defaults *cpu_def;
2709 int rc = 0;
2710 2547
2711 if (no_load) 2548 if (no_load)
2712 return -ENODEV; 2549 return -ENODEV;
2713 2550
2714 if (x86_match_cpu(hwp_support_ids) && !no_hwp) { 2551 if (x86_match_cpu(hwp_support_ids)) {
2715 copy_cpu_funcs(&core_params.funcs); 2552 copy_cpu_funcs(&core_funcs);
2716 hwp_active++; 2553 if (no_hwp) {
2717 intel_pstate.attr = hwp_cpufreq_attrs; 2554 pstate_funcs.update_util = intel_pstate_update_util;
2718 goto hwp_cpu_matched; 2555 } else {
2719 } 2556 hwp_active++;
2720 2557 intel_pstate.attr = hwp_cpufreq_attrs;
2721 id = x86_match_cpu(intel_pstate_cpu_ids); 2558 pstate_funcs.update_util = intel_pstate_update_util_hwp;
2722 if (!id) 2559 goto hwp_cpu_matched;
2723 return -ENODEV; 2560 }
2561 } else {
2562 const struct x86_cpu_id *id;
2724 2563
2725 cpu_def = (struct cpu_defaults *)id->driver_data; 2564 id = x86_match_cpu(intel_pstate_cpu_ids);
2565 if (!id)
2566 return -ENODEV;
2726 2567
2727 copy_pid_params(&cpu_def->pid_policy); 2568 copy_cpu_funcs((struct pstate_funcs *)id->driver_data);
2728 copy_cpu_funcs(&cpu_def->funcs); 2569 }
2729 2570
2730 if (intel_pstate_msrs_not_valid()) 2571 if (intel_pstate_msrs_not_valid())
2731 return -ENODEV; 2572 return -ENODEV;
@@ -2752,7 +2593,7 @@ hwp_cpu_matched:
2752 intel_pstate_sysfs_expose_params(); 2593 intel_pstate_sysfs_expose_params();
2753 2594
2754 mutex_lock(&intel_pstate_driver_lock); 2595 mutex_lock(&intel_pstate_driver_lock);
2755 rc = intel_pstate_register_driver(); 2596 rc = intel_pstate_register_driver(default_driver);
2756 mutex_unlock(&intel_pstate_driver_lock); 2597 mutex_unlock(&intel_pstate_driver_lock);
2757 if (rc) 2598 if (rc)
2758 return rc; 2599 return rc;
@@ -2773,7 +2614,7 @@ static int __init intel_pstate_setup(char *str)
2773 no_load = 1; 2614 no_load = 1;
2774 } else if (!strcmp(str, "passive")) { 2615 } else if (!strcmp(str, "passive")) {
2775 pr_info("Passive mode enabled\n"); 2616 pr_info("Passive mode enabled\n");
2776 intel_pstate_driver = &intel_cpufreq; 2617 default_driver = &intel_cpufreq;
2777 no_hwp = 1; 2618 no_hwp = 1;
2778 } 2619 }
2779 if (!strcmp(str, "no_hwp")) { 2620 if (!strcmp(str, "no_hwp")) {
generated by cgit v1.2.2 (git 2.25.0) at 2025-06-01 05:01:59 -0400