aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRafael J. Wysocki <rafael.j.wysocki@intel.com>2016-07-25 07:46:08 -0400
committerRafael J. Wysocki <rafael.j.wysocki@intel.com>2016-07-25 07:46:08 -0400
commit9def970eada83e6800cb4612ffdcd8eb7908c7b4 (patch)
tree082dd6127f6e5c69e588c6615507f5ccc847ff26
parent9fedbb3b6bb8d00c973f01dcf14af8eedf5eb495 (diff)
parentda7d3abe1c9e5ebac2cf86f97e9e89888a5e2094 (diff)
Merge branch 'pm-cpufreq'
* pm-cpufreq: (41 commits) Revert "cpufreq: pcc-cpufreq: update default value of cpuinfo_transition_latency" cpufreq: export cpufreq_driver_resolve_freq() cpufreq: Disallow ->resolve_freq() for drivers providing ->target_index() cpufreq: acpi-cpufreq: use cached frequency mapping when possible cpufreq: schedutil: map raw required frequency to driver frequency cpufreq: add cpufreq_driver_resolve_freq() cpufreq: intel_pstate: Check cpuid for MSR_HWP_INTERRUPT intel_pstate: Update cpu_frequency tracepoint every time cpufreq: intel_pstate: clean remnant struct element cpufreq: powernv: Replacing pstate_id with frequency table index intel_pstate: Fix MSR_CONFIG_TDP_x addressing in core_get_max_pstate() cpufreq: Reuse new freq-table helpers cpufreq: Handle sorted frequency tables more efficiently cpufreq: Drop redundant check from cpufreq_update_current_freq() intel_pstate: Declare pid_params/pstate_funcs/hwp_active __read_mostly intel_pstate: add __init/__initdata marker to some functions/variables intel_pstate: Fix incorrect placement of __initdata cpufreq: mvebu: fix integer to pointer cast cpufreq: intel_pstate: Broxton support cpufreq: conservative: Do not use transition notifications ...
Diffstat
-rw-r--r--Documentation/cpu-freq/core.txt4
-rw-r--r--Documentation/cpu-freq/cpu-drivers.txt10
-rw-r--r--Documentation/cpu-freq/pcc-cpufreq.txt4
-rw-r--r--arch/powerpc/platforms/cell/cpufreq_spudemand.c72
-rw-r--r--drivers/cpufreq/Kconfig13
-rw-r--r--drivers/cpufreq/acpi-cpufreq.c17
-rw-r--r--drivers/cpufreq/amd_freq_sensitivity.c10
-rw-r--r--drivers/cpufreq/cpufreq.c223
-rw-r--r--drivers/cpufreq/cpufreq_conservative.c88
-rw-r--r--drivers/cpufreq/cpufreq_governor.c73
-rw-r--r--drivers/cpufreq/cpufreq_governor.h24
-rw-r--r--drivers/cpufreq/cpufreq_ondemand.c38
-rw-r--r--drivers/cpufreq/cpufreq_ondemand.h1
-rw-r--r--drivers/cpufreq/cpufreq_performance.c19
-rw-r--r--drivers/cpufreq/cpufreq_powersave.c19
-rw-r--r--drivers/cpufreq/cpufreq_stats.c157
-rw-r--r--drivers/cpufreq/cpufreq_userspace.c104
-rw-r--r--drivers/cpufreq/davinci-cpufreq.c22
-rw-r--r--drivers/cpufreq/freq_table.c106
-rw-r--r--drivers/cpufreq/intel_pstate.c70
-rw-r--r--drivers/cpufreq/mvebu-cpufreq.c2
-rw-r--r--drivers/cpufreq/pcc-cpufreq.c2
-rw-r--r--drivers/cpufreq/powernv-cpufreq.c181
-rw-r--r--drivers/cpufreq/ppc_cbe_cpufreq_pmi.c3
-rw-r--r--drivers/cpufreq/s3c24xx-cpufreq.c33
-rw-r--r--drivers/cpufreq/s5pv210-cpufreq.c7
-rw-r--r--drivers/thermal/cpu_cooling.c24
-rw-r--r--include/linux/cpufreq.h289
-rw-r--r--kernel/sched/cpufreq_schedutil.c74
29 files changed, 908 insertions, 781 deletions
diff --git a/Documentation/cpu-freq/core.txt b/Documentation/cpu-freq/core.txt
index ba78e7c2a069..4bc7287806de 100644
--- a/Documentation/cpu-freq/core.txt
+++ b/Documentation/cpu-freq/core.txt
@@ -96,7 +96,7 @@ new - new frequency
96For details about OPP, see Documentation/power/opp.txt 96For details about OPP, see Documentation/power/opp.txt
97 97
98dev_pm_opp_init_cpufreq_table - cpufreq framework typically is initialized with 98dev_pm_opp_init_cpufreq_table - cpufreq framework typically is initialized with
99 cpufreq_frequency_table_cpuinfo which is provided with the list of 99 cpufreq_table_validate_and_show() which is provided with the list of
100 frequencies that are available for operation. This function provides 100 frequencies that are available for operation. This function provides
101 a ready to use conversion routine to translate the OPP layer's internal 101 a ready to use conversion routine to translate the OPP layer's internal
102 information about the available frequencies into a format readily 102 information about the available frequencies into a format readily
@@ -110,7 +110,7 @@ dev_pm_opp_init_cpufreq_table - cpufreq framework typically is initialized with
110 /* Do things */ 110 /* Do things */
111 r = dev_pm_opp_init_cpufreq_table(dev, &freq_table); 111 r = dev_pm_opp_init_cpufreq_table(dev, &freq_table);
112 if (!r) 112 if (!r)
113 cpufreq_frequency_table_cpuinfo(policy, freq_table); 113 cpufreq_table_validate_and_show(policy, freq_table);
114 /* Do other things */ 114 /* Do other things */
115 } 115 }
116 116
diff --git a/Documentation/cpu-freq/cpu-drivers.txt b/Documentation/cpu-freq/cpu-drivers.txt
index 14f4e6336d88..772b94fde264 100644
--- a/Documentation/cpu-freq/cpu-drivers.txt
+++ b/Documentation/cpu-freq/cpu-drivers.txt
@@ -231,7 +231,7 @@ if you want to skip one entry in the table, set the frequency to
231CPUFREQ_ENTRY_INVALID. The entries don't need to be in ascending 231CPUFREQ_ENTRY_INVALID. The entries don't need to be in ascending
232order. 232order.
233 233
234By calling cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, 234By calling cpufreq_table_validate_and_show(struct cpufreq_policy *policy,
235 struct cpufreq_frequency_table *table); 235 struct cpufreq_frequency_table *table);
236the cpuinfo.min_freq and cpuinfo.max_freq values are detected, and 236the cpuinfo.min_freq and cpuinfo.max_freq values are detected, and
237policy->min and policy->max are set to the same values. This is 237policy->min and policy->max are set to the same values. This is
@@ -244,14 +244,12 @@ policy->max, and all other criteria are met. This is helpful for the
244->verify call. 244->verify call.
245 245
246int cpufreq_frequency_table_target(struct cpufreq_policy *policy, 246int cpufreq_frequency_table_target(struct cpufreq_policy *policy,
247 struct cpufreq_frequency_table *table,
248 unsigned int target_freq, 247 unsigned int target_freq,
249 unsigned int relation, 248 unsigned int relation);
250 unsigned int *index);
251 249
252is the corresponding frequency table helper for the ->target 250is the corresponding frequency table helper for the ->target
253stage. Just pass the values to this function, and the unsigned int 251stage. Just pass the values to this function, and this function
254index returns the number of the frequency table entry which contains 252returns the number of the frequency table entry which contains
255the frequency the CPU shall be set to. 253the frequency the CPU shall be set to.
256 254
257The following macros can be used as iterators over cpufreq_frequency_table: 255The following macros can be used as iterators over cpufreq_frequency_table:
diff --git a/Documentation/cpu-freq/pcc-cpufreq.txt b/Documentation/cpu-freq/pcc-cpufreq.txt
index 0a94224ad296..9e3c3b33514c 100644
--- a/Documentation/cpu-freq/pcc-cpufreq.txt
+++ b/Documentation/cpu-freq/pcc-cpufreq.txt
@@ -159,8 +159,8 @@ to be strictly associated with a P-state.
159 159
1602.2 cpuinfo_transition_latency: 1602.2 cpuinfo_transition_latency:
161------------------------------- 161-------------------------------
162The cpuinfo_transition_latency field is CPUFREQ_ETERNAL. The PCC specification 162The cpuinfo_transition_latency field is 0. The PCC specification does
163does not include a field to expose this value currently. 163not include a field to expose this value currently.
164 164
1652.3 cpuinfo_cur_freq: 1652.3 cpuinfo_cur_freq:
166--------------------- 166---------------------
diff --git a/arch/powerpc/platforms/cell/cpufreq_spudemand.c b/arch/powerpc/platforms/cell/cpufreq_spudemand.c
index 82607d621aca..88301e53f085 100644
--- a/arch/powerpc/platforms/cell/cpufreq_spudemand.c
+++ b/arch/powerpc/platforms/cell/cpufreq_spudemand.c
@@ -85,61 +85,57 @@ static void spu_gov_cancel_work(struct spu_gov_info_struct *info)
85 cancel_delayed_work_sync(&info->work); 85 cancel_delayed_work_sync(&info->work);
86} 86}
87 87
88static int spu_gov_govern(struct cpufreq_policy *policy, unsigned int event) 88static int spu_gov_start(struct cpufreq_policy *policy)
89{ 89{
90 unsigned int cpu = policy->cpu; 90 unsigned int cpu = policy->cpu;
91 struct spu_gov_info_struct *info, *affected_info; 91 struct spu_gov_info_struct *info = &per_cpu(spu_gov_info, cpu);
92 struct spu_gov_info_struct *affected_info;
92 int i; 93 int i;
93 int ret = 0;
94 94
95 info = &per_cpu(spu_gov_info, cpu); 95 if (!cpu_online(cpu)) {
96 96 printk(KERN_ERR "cpu %d is not online\n", cpu);
97 switch (event) { 97 return -EINVAL;
98 case CPUFREQ_GOV_START: 98 }
99 if (!cpu_online(cpu)) {
100 printk(KERN_ERR "cpu %d is not online\n", cpu);
101 ret = -EINVAL;
102 break;
103 }
104 99
105 if (!policy->cur) { 100 if (!policy->cur) {
106 printk(KERN_ERR "no cpu specified in policy\n"); 101 printk(KERN_ERR "no cpu specified in policy\n");
107 ret = -EINVAL; 102 return -EINVAL;
108 break; 103 }
109 }
110 104
111 /* initialize spu_gov_info for all affected cpus */ 105 /* initialize spu_gov_info for all affected cpus */
112 for_each_cpu(i, policy->cpus) { 106 for_each_cpu(i, policy->cpus) {
113 affected_info = &per_cpu(spu_gov_info, i); 107 affected_info = &per_cpu(spu_gov_info, i);
114 affected_info->policy = policy; 108 affected_info->policy = policy;
115 } 109 }
116 110
117 info->poll_int = POLL_TIME; 111 info->poll_int = POLL_TIME;
118 112
119 /* setup timer */ 113 /* setup timer */
120 spu_gov_init_work(info); 114 spu_gov_init_work(info);
121 115
122 break; 116 return 0;
117}
123 118
124 case CPUFREQ_GOV_STOP: 119static void spu_gov_stop(struct cpufreq_policy *policy)
125 /* cancel timer */ 120{
126 spu_gov_cancel_work(info); 121 unsigned int cpu = policy->cpu;
122 struct spu_gov_info_struct *info = &per_cpu(spu_gov_info, cpu);
123 int i;
127 124
128 /* clean spu_gov_info for all affected cpus */ 125 /* cancel timer */
129 for_each_cpu (i, policy->cpus) { 126 spu_gov_cancel_work(info);
130 info = &per_cpu(spu_gov_info, i);
131 info->policy = NULL;
132 }
133 127
134 break; 128 /* clean spu_gov_info for all affected cpus */
129 for_each_cpu (i, policy->cpus) {
130 info = &per_cpu(spu_gov_info, i);
131 info->policy = NULL;
135 } 132 }
136
137 return ret;
138} 133}
139 134
140static struct cpufreq_governor spu_governor = { 135static struct cpufreq_governor spu_governor = {
141 .name = "spudemand", 136 .name = "spudemand",
142 .governor = spu_gov_govern, 137 .start = spu_gov_start,
138 .stop = spu_gov_stop,
143 .owner = THIS_MODULE, 139 .owner = THIS_MODULE,
144}; 140};
145 141
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index b7445b6ae5a4..c822d72629d5 100644
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -31,23 +31,18 @@ config CPU_FREQ_BOOST_SW
31 depends on THERMAL 31 depends on THERMAL
32 32
33config CPU_FREQ_STAT 33config CPU_FREQ_STAT
34 tristate "CPU frequency translation statistics" 34 bool "CPU frequency transition statistics"
35 default y 35 default y
36 help 36 help
37 This driver exports CPU frequency statistics information through sysfs 37 Export CPU frequency statistics information through sysfs.
38 file system.
39
40 To compile this driver as a module, choose M here: the
41 module will be called cpufreq_stats.
42 38
43 If in doubt, say N. 39 If in doubt, say N.
44 40
45config CPU_FREQ_STAT_DETAILS 41config CPU_FREQ_STAT_DETAILS
46 bool "CPU frequency translation statistics details" 42 bool "CPU frequency transition statistics details"
47 depends on CPU_FREQ_STAT 43 depends on CPU_FREQ_STAT
48 help 44 help
49 This will show detail CPU frequency translation table in sysfs file 45 Show detailed CPU frequency transition table in sysfs.
50 system.
51 46
52 If in doubt, say N. 47 If in doubt, say N.
53 48
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
index 32a15052f363..297e9128fe9f 100644
--- a/drivers/cpufreq/acpi-cpufreq.c
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -468,20 +468,17 @@ unsigned int acpi_cpufreq_fast_switch(struct cpufreq_policy *policy,
468 struct acpi_cpufreq_data *data = policy->driver_data; 468 struct acpi_cpufreq_data *data = policy->driver_data;
469 struct acpi_processor_performance *perf; 469 struct acpi_processor_performance *perf;
470 struct cpufreq_frequency_table *entry; 470 struct cpufreq_frequency_table *entry;
471 unsigned int next_perf_state, next_freq, freq; 471 unsigned int next_perf_state, next_freq, index;
472 472
473 /* 473 /*
474 * Find the closest frequency above target_freq. 474 * Find the closest frequency above target_freq.
475 *
476 * The table is sorted in the reverse order with respect to the
477 * frequency and all of the entries are valid (see the initialization).
478 */ 475 */
479 entry = policy->freq_table; 476 if (policy->cached_target_freq == target_freq)
480 do { 477 index = policy->cached_resolved_idx;
481 entry++; 478 else
482 freq = entry->frequency; 479 index = cpufreq_table_find_index_dl(policy, target_freq);
483 } while (freq >= target_freq && freq != CPUFREQ_TABLE_END); 480
484 entry--; 481 entry = &policy->freq_table[index];
485 next_freq = entry->frequency; 482 next_freq = entry->frequency;
486 next_perf_state = entry->driver_data; 483 next_perf_state = entry->driver_data;
487 484
diff --git a/drivers/cpufreq/amd_freq_sensitivity.c b/drivers/cpufreq/amd_freq_sensitivity.c
index 404360cad25c..042023bbbf62 100644
--- a/drivers/cpufreq/amd_freq_sensitivity.c
+++ b/drivers/cpufreq/amd_freq_sensitivity.c
@@ -48,9 +48,8 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy,
48 struct policy_dbs_info *policy_dbs = policy->governor_data; 48 struct policy_dbs_info *policy_dbs = policy->governor_data;
49 struct dbs_data *od_data = policy_dbs->dbs_data; 49 struct dbs_data *od_data = policy_dbs->dbs_data;
50 struct od_dbs_tuners *od_tuners = od_data->tuners; 50 struct od_dbs_tuners *od_tuners = od_data->tuners;
51 struct od_policy_dbs_info *od_info = to_dbs_info(policy_dbs);
52 51
53 if (!od_info->freq_table) 52 if (!policy->freq_table)
54 return freq_next; 53 return freq_next;
55 54
56 rdmsr_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, 55 rdmsr_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_ACTUAL,
@@ -92,10 +91,9 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy,
92 else { 91 else {
93 unsigned int index; 92 unsigned int index;
94 93
95 cpufreq_frequency_table_target(policy, 94 index = cpufreq_table_find_index_h(policy,
96 od_info->freq_table, policy->cur - 1, 95 policy->cur - 1);
97 CPUFREQ_RELATION_H, &index); 96 freq_next = policy->freq_table[index].frequency;
98 freq_next = od_info->freq_table[index].frequency;
99 } 97 }
100 98
101 data->freq_prev = freq_next; 99 data->freq_prev = freq_next;
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 5617c7087d77..3dd4884c6f9e 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -74,19 +74,12 @@ static inline bool has_target(void)
74} 74}
75 75
76/* internal prototypes */ 76/* internal prototypes */
77static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
78static unsigned int __cpufreq_get(struct cpufreq_policy *policy); 77static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
78static int cpufreq_init_governor(struct cpufreq_policy *policy);
79static void cpufreq_exit_governor(struct cpufreq_policy *policy);
79static int cpufreq_start_governor(struct cpufreq_policy *policy); 80static int cpufreq_start_governor(struct cpufreq_policy *policy);
80 81static void cpufreq_stop_governor(struct cpufreq_policy *policy);
81static inline void cpufreq_exit_governor(struct cpufreq_policy *policy) 82static void cpufreq_governor_limits(struct cpufreq_policy *policy);
82{
83 (void)cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
84}
85
86static inline void cpufreq_stop_governor(struct cpufreq_policy *policy)
87{
88 (void)cpufreq_governor(policy, CPUFREQ_GOV_STOP);
89}
90 83
91/** 84/**
92 * Two notifier lists: the "policy" list is involved in the 85 * Two notifier lists: the "policy" list is involved in the
@@ -133,15 +126,6 @@ struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
133} 126}
134EXPORT_SYMBOL_GPL(get_governor_parent_kobj); 127EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
135 128
136struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
137{
138 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
139
140 return policy && !policy_is_inactive(policy) ?
141 policy->freq_table : NULL;
142}
143EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);
144
145static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) 129static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
146{ 130{
147 u64 idle_time; 131 u64 idle_time;
@@ -354,6 +338,7 @@ static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
354 pr_debug("FREQ: %lu - CPU: %lu\n", 338 pr_debug("FREQ: %lu - CPU: %lu\n",
355 (unsigned long)freqs->new, (unsigned long)freqs->cpu); 339 (unsigned long)freqs->new, (unsigned long)freqs->cpu);
356 trace_cpu_frequency(freqs->new, freqs->cpu); 340 trace_cpu_frequency(freqs->new, freqs->cpu);
341 cpufreq_stats_record_transition(policy, freqs->new);
357 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 342 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
358 CPUFREQ_POSTCHANGE, freqs); 343 CPUFREQ_POSTCHANGE, freqs);
359 if (likely(policy) && likely(policy->cpu == freqs->cpu)) 344 if (likely(policy) && likely(policy->cpu == freqs->cpu))
@@ -507,6 +492,38 @@ void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
507} 492}
508EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); 493EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
509 494
495/**
496 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
497 * one.
498 * @target_freq: target frequency to resolve.
499 *
500 * The target to driver frequency mapping is cached in the policy.
501 *
502 * Return: Lowest driver-supported frequency greater than or equal to the
503 * given target_freq, subject to policy (min/max) and driver limitations.
504 */
505unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
506 unsigned int target_freq)
507{
508 target_freq = clamp_val(target_freq, policy->min, policy->max);
509 policy->cached_target_freq = target_freq;
510
511 if (cpufreq_driver->target_index) {
512 int idx;
513
514 idx = cpufreq_frequency_table_target(policy, target_freq,
515 CPUFREQ_RELATION_L);
516 policy->cached_resolved_idx = idx;
517 return policy->freq_table[idx].frequency;
518 }
519
520 if (cpufreq_driver->resolve_freq)
521 return cpufreq_driver->resolve_freq(policy, target_freq);
522
523 return target_freq;
524}
525EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
526
510/********************************************************************* 527/*********************************************************************
511 * SYSFS INTERFACE * 528 * SYSFS INTERFACE *
512 *********************************************************************/ 529 *********************************************************************/
@@ -1115,6 +1132,7 @@ static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)
1115 CPUFREQ_REMOVE_POLICY, policy); 1132 CPUFREQ_REMOVE_POLICY, policy);
1116 1133
1117 down_write(&policy->rwsem); 1134 down_write(&policy->rwsem);
1135 cpufreq_stats_free_table(policy);
1118 cpufreq_remove_dev_symlink(policy); 1136 cpufreq_remove_dev_symlink(policy);
1119 kobj = &policy->kobj; 1137 kobj = &policy->kobj;
1120 cmp = &policy->kobj_unregister; 1138 cmp = &policy->kobj_unregister;
@@ -1265,13 +1283,12 @@ static int cpufreq_online(unsigned int cpu)
1265 } 1283 }
1266 } 1284 }
1267 1285
1268 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1269 CPUFREQ_START, policy);
1270
1271 if (new_policy) { 1286 if (new_policy) {
1272 ret = cpufreq_add_dev_interface(policy); 1287 ret = cpufreq_add_dev_interface(policy);
1273 if (ret) 1288 if (ret)
1274 goto out_exit_policy; 1289 goto out_exit_policy;
1290
1291 cpufreq_stats_create_table(policy);
1275 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1292 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1276 CPUFREQ_CREATE_POLICY, policy); 1293 CPUFREQ_CREATE_POLICY, policy);
1277 1294
@@ -1280,6 +1297,9 @@ static int cpufreq_online(unsigned int cpu)
1280 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1297 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1281 } 1298 }
1282 1299
1300 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1301 CPUFREQ_START, policy);
1302
1283 ret = cpufreq_init_policy(policy); 1303 ret = cpufreq_init_policy(policy);
1284 if (ret) { 1304 if (ret) {
1285 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", 1305 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
@@ -1556,9 +1576,6 @@ static unsigned int cpufreq_update_current_freq(struct cpufreq_policy *policy)
1556{ 1576{
1557 unsigned int new_freq; 1577 unsigned int new_freq;
1558 1578
1559 if (cpufreq_suspended)
1560 return 0;
1561
1562 new_freq = cpufreq_driver->get(policy->cpu); 1579 new_freq = cpufreq_driver->get(policy->cpu);
1563 if (!new_freq) 1580 if (!new_freq)
1564 return 0; 1581 return 0;
@@ -1864,14 +1881,17 @@ static int __target_intermediate(struct cpufreq_policy *policy,
1864 return ret; 1881 return ret;
1865} 1882}
1866 1883
1867static int __target_index(struct cpufreq_policy *policy, 1884static int __target_index(struct cpufreq_policy *policy, int index)
1868 struct cpufreq_frequency_table *freq_table, int index)
1869{ 1885{
1870 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; 1886 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1871 unsigned int intermediate_freq = 0; 1887 unsigned int intermediate_freq = 0;
1888 unsigned int newfreq = policy->freq_table[index].frequency;
1872 int retval = -EINVAL; 1889 int retval = -EINVAL;
1873 bool notify; 1890 bool notify;
1874 1891
1892 if (newfreq == policy->cur)
1893 return 0;
1894
1875 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); 1895 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1876 if (notify) { 1896 if (notify) {
1877 /* Handle switching to intermediate frequency */ 1897 /* Handle switching to intermediate frequency */
@@ -1886,7 +1906,7 @@ static int __target_index(struct cpufreq_policy *policy,
1886 freqs.old = freqs.new; 1906 freqs.old = freqs.new;
1887 } 1907 }
1888 1908
1889 freqs.new = freq_table[index].frequency; 1909 freqs.new = newfreq;
1890 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", 1910 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1891 __func__, policy->cpu, freqs.old, freqs.new); 1911 __func__, policy->cpu, freqs.old, freqs.new);
1892 1912
@@ -1923,17 +1943,13 @@ int __cpufreq_driver_target(struct cpufreq_policy *policy,
1923 unsigned int relation) 1943 unsigned int relation)
1924{ 1944{
1925 unsigned int old_target_freq = target_freq; 1945 unsigned int old_target_freq = target_freq;
1926 struct cpufreq_frequency_table *freq_table; 1946 int index;
1927 int index, retval;
1928 1947
1929 if (cpufreq_disabled()) 1948 if (cpufreq_disabled())
1930 return -ENODEV; 1949 return -ENODEV;
1931 1950
1932 /* Make sure that target_freq is within supported range */ 1951 /* Make sure that target_freq is within supported range */
1933 if (target_freq > policy->max) 1952 target_freq = clamp_val(target_freq, policy->min, policy->max);
1934 target_freq = policy->max;
1935 if (target_freq < policy->min)
1936 target_freq = policy->min;
1937 1953
1938 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", 1954 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1939 policy->cpu, target_freq, relation, old_target_freq); 1955 policy->cpu, target_freq, relation, old_target_freq);
@@ -1956,23 +1972,9 @@ int __cpufreq_driver_target(struct cpufreq_policy *policy,
1956 if (!cpufreq_driver->target_index) 1972 if (!cpufreq_driver->target_index)
1957 return -EINVAL; 1973 return -EINVAL;
1958 1974
1959 freq_table = cpufreq_frequency_get_table(policy->cpu); 1975 index = cpufreq_frequency_table_target(policy, target_freq, relation);
1960 if (unlikely(!freq_table)) {
1961 pr_err("%s: Unable to find freq_table\n", __func__);
1962 return -EINVAL;
1963 }
1964
1965 retval = cpufreq_frequency_table_target(policy, freq_table, target_freq,
1966 relation, &index);
1967 if (unlikely(retval)) {
1968 pr_err("%s: Unable to find matching freq\n", __func__);
1969 return retval;
1970 }
1971
1972 if (freq_table[index].frequency == policy->cur)
1973 return 0;
1974 1976
1975 return __target_index(policy, freq_table, index); 1977 return __target_index(policy, index);
1976} 1978}
1977EXPORT_SYMBOL_GPL(__cpufreq_driver_target); 1979EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1978 1980
@@ -1997,7 +1999,7 @@ __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
1997 return NULL; 1999 return NULL;
1998} 2000}
1999 2001
2000static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event) 2002static int cpufreq_init_governor(struct cpufreq_policy *policy)
2001{ 2003{
2002 int ret; 2004 int ret;
2003 2005
@@ -2025,36 +2027,82 @@ static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event)
2025 } 2027 }
2026 } 2028 }
2027 2029
2028 if (event == CPUFREQ_GOV_POLICY_INIT) 2030 if (!try_module_get(policy->governor->owner))
2029 if (!try_module_get(policy->governor->owner)) 2031 return -EINVAL;
2030 return -EINVAL;
2031
2032 pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event);
2033 2032
2034 ret = policy->governor->governor(policy, event); 2033 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2035 2034
2036 if (event == CPUFREQ_GOV_POLICY_INIT) { 2035 if (policy->governor->init) {
2037 if (ret) 2036 ret = policy->governor->init(policy);
2037 if (ret) {
2038 module_put(policy->governor->owner); 2038 module_put(policy->governor->owner);
2039 else 2039 return ret;
2040 policy->governor->initialized++; 2040 }
2041 } else if (event == CPUFREQ_GOV_POLICY_EXIT) {
2042 policy->governor->initialized--;
2043 module_put(policy->governor->owner);
2044 } 2041 }
2045 2042
2046 return ret; 2043 return 0;
2044}
2045
2046static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2047{
2048 if (cpufreq_suspended || !policy->governor)
2049 return;
2050
2051 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2052
2053 if (policy->governor->exit)
2054 policy->governor->exit(policy);
2055
2056 module_put(policy->governor->owner);
2047} 2057}
2048 2058
2049static int cpufreq_start_governor(struct cpufreq_policy *policy) 2059static int cpufreq_start_governor(struct cpufreq_policy *policy)
2050{ 2060{
2051 int ret; 2061 int ret;
2052 2062
2063 if (cpufreq_suspended)
2064 return 0;
2065
2066 if (!policy->governor)
2067 return -EINVAL;
2068
2069 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2070
2053 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) 2071 if (cpufreq_driver->get && !cpufreq_driver->setpolicy)
2054 cpufreq_update_current_freq(policy); 2072 cpufreq_update_current_freq(policy);
2055 2073
2056 ret = cpufreq_governor(policy, CPUFREQ_GOV_START); 2074 if (policy->governor->start) {
2057 return ret ? ret : cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 2075 ret = policy->governor->start(policy);
2076 if (ret)
2077 return ret;
2078 }
2079
2080 if (policy->governor->limits)
2081 policy->governor->limits(policy);
2082
2083 return 0;
2084}
2085
2086static void cpufreq_stop_governor(struct cpufreq_policy *policy)
2087{
2088 if (cpufreq_suspended || !policy->governor)
2089 return;
2090
2091 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2092
2093 if (policy->governor->stop)
2094 policy->governor->stop(policy);
2095}
2096
2097static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2098{
2099 if (cpufreq_suspended || !policy->governor)
2100 return;
2101
2102 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2103
2104 if (policy->governor->limits)
2105 policy->governor->limits(policy);
2058} 2106}
2059 2107
2060int cpufreq_register_governor(struct cpufreq_governor *governor) 2108int cpufreq_register_governor(struct cpufreq_governor *governor)
@@ -2069,7 +2117,6 @@ int cpufreq_register_governor(struct cpufreq_governor *governor)
2069 2117
2070 mutex_lock(&cpufreq_governor_mutex); 2118 mutex_lock(&cpufreq_governor_mutex);
2071 2119
2072 governor->initialized = 0;
2073 err = -EBUSY; 2120 err = -EBUSY;
2074 if (!find_governor(governor->name)) { 2121 if (!find_governor(governor->name)) {
2075 err = 0; 2122 err = 0;
@@ -2184,6 +2231,8 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2184 policy->min = new_policy->min; 2231 policy->min = new_policy->min;
2185 policy->max = new_policy->max; 2232 policy->max = new_policy->max;
2186 2233
2234 policy->cached_target_freq = UINT_MAX;
2235
2187 pr_debug("new min and max freqs are %u - %u kHz\n", 2236 pr_debug("new min and max freqs are %u - %u kHz\n",
2188 policy->min, policy->max); 2237 policy->min, policy->max);
2189 2238
@@ -2195,7 +2244,8 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2195 2244
2196 if (new_policy->governor == policy->governor) { 2245 if (new_policy->governor == policy->governor) {
2197 pr_debug("cpufreq: governor limits update\n"); 2246 pr_debug("cpufreq: governor limits update\n");
2198 return cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 2247 cpufreq_governor_limits(policy);
2248 return 0;
2199 } 2249 }
2200 2250
2201 pr_debug("governor switch\n"); 2251 pr_debug("governor switch\n");
@@ -2210,7 +2260,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2210 2260
2211 /* start new governor */ 2261 /* start new governor */
2212 policy->governor = new_policy->governor; 2262 policy->governor = new_policy->governor;
2213 ret = cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT); 2263 ret = cpufreq_init_governor(policy);
2214 if (!ret) { 2264 if (!ret) {
2215 ret = cpufreq_start_governor(policy); 2265 ret = cpufreq_start_governor(policy);
2216 if (!ret) { 2266 if (!ret) {
@@ -2224,7 +2274,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2224 pr_debug("starting governor %s failed\n", policy->governor->name); 2274 pr_debug("starting governor %s failed\n", policy->governor->name);
2225 if (old_gov) { 2275 if (old_gov) {
2226 policy->governor = old_gov; 2276 policy->governor = old_gov;
2227 if (cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) 2277 if (cpufreq_init_governor(policy))
2228 policy->governor = NULL; 2278 policy->governor = NULL;
2229 else 2279 else
2230 cpufreq_start_governor(policy); 2280 cpufreq_start_governor(policy);
@@ -2309,26 +2359,25 @@ static struct notifier_block __refdata cpufreq_cpu_notifier = {
2309 *********************************************************************/ 2359 *********************************************************************/
2310static int cpufreq_boost_set_sw(int state) 2360static int cpufreq_boost_set_sw(int state)
2311{ 2361{
2312 struct cpufreq_frequency_table *freq_table;
2313 struct cpufreq_policy *policy; 2362 struct cpufreq_policy *policy;
2314 int ret = -EINVAL; 2363 int ret = -EINVAL;
2315 2364
2316 for_each_active_policy(policy) { 2365 for_each_active_policy(policy) {
2317 freq_table = cpufreq_frequency_get_table(policy->cpu); 2366 if (!policy->freq_table)
2318 if (freq_table) { 2367 continue;
2319 ret = cpufreq_frequency_table_cpuinfo(policy, 2368
2320 freq_table); 2369 ret = cpufreq_frequency_table_cpuinfo(policy,
2321 if (ret) { 2370 policy->freq_table);
2322 pr_err("%s: Policy frequency update failed\n", 2371 if (ret) {
2323 __func__); 2372 pr_err("%s: Policy frequency update failed\n",
2324 break; 2373 __func__);
2325 } 2374 break;
2326
2327 down_write(&policy->rwsem);
2328 policy->user_policy.max = policy->max;
2329 cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2330 up_write(&policy->rwsem);
2331 } 2375 }
2376
2377 down_write(&policy->rwsem);
2378 policy->user_policy.max = policy->max;
2379 cpufreq_governor_limits(policy);
2380 up_write(&policy->rwsem);
2332 } 2381 }
2333 2382
2334 return ret; 2383 return ret;
diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c
index 316df247e00d..18da4f8051d3 100644
--- a/drivers/cpufreq/cpufreq_conservative.c
+++ b/drivers/cpufreq/cpufreq_conservative.c
@@ -17,7 +17,6 @@
17struct cs_policy_dbs_info { 17struct cs_policy_dbs_info {
18 struct policy_dbs_info policy_dbs; 18 struct policy_dbs_info policy_dbs;
19 unsigned int down_skip; 19 unsigned int down_skip;
20 unsigned int requested_freq;
21}; 20};
22 21
23static inline struct cs_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs) 22static inline struct cs_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs)
@@ -75,19 +74,17 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy)
75 74
76 /* Check for frequency increase */ 75 /* Check for frequency increase */
77 if (load > dbs_data->up_threshold) { 76 if (load > dbs_data->up_threshold) {
77 unsigned int requested_freq = policy->cur;
78
78 dbs_info->down_skip = 0; 79 dbs_info->down_skip = 0;
79 80
80 /* if we are already at full speed then break out early */ 81 /* if we are already at full speed then break out early */
81 if (dbs_info->requested_freq == policy->max) 82 if (requested_freq == policy->max)
82 goto out; 83 goto out;
83 84
84 dbs_info->requested_freq += get_freq_target(cs_tuners, policy); 85 requested_freq += get_freq_target(cs_tuners, policy);
85
86 if (dbs_info->requested_freq > policy->max)
87 dbs_info->requested_freq = policy->max;
88 86
89 __cpufreq_driver_target(policy, dbs_info->requested_freq, 87 __cpufreq_driver_target(policy, requested_freq, CPUFREQ_RELATION_H);
90 CPUFREQ_RELATION_H);
91 goto out; 88 goto out;
92 } 89 }
93 90
@@ -98,36 +95,27 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy)
98 95
99 /* Check for frequency decrease */ 96 /* Check for frequency decrease */
100 if (load < cs_tuners->down_threshold) { 97 if (load < cs_tuners->down_threshold) {
101 unsigned int freq_target; 98 unsigned int freq_target, requested_freq = policy->cur;
102 /* 99 /*
103 * if we cannot reduce the frequency anymore, break out early 100 * if we cannot reduce the frequency anymore, break out early
104 */ 101 */
105 if (policy->cur == policy->min) 102 if (requested_freq == policy->min)
106 goto out; 103 goto out;
107 104
108 freq_target = get_freq_target(cs_tuners, policy); 105 freq_target = get_freq_target(cs_tuners, policy);
109 if (dbs_info->requested_freq > freq_target) 106 if (requested_freq > freq_target)
110 dbs_info->requested_freq -= freq_target; 107 requested_freq -= freq_target;
111 else 108 else
112 dbs_info->requested_freq = policy->min; 109 requested_freq = policy->min;
113 110
114 __cpufreq_driver_target(policy, dbs_info->requested_freq, 111 __cpufreq_driver_target(policy, requested_freq, CPUFREQ_RELATION_L);
115 CPUFREQ_RELATION_L);
116 } 112 }
117 113
118 out: 114 out:
119 return dbs_data->sampling_rate; 115 return dbs_data->sampling_rate;
120} 116}
121 117
122static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
123 void *data);
124
125static struct notifier_block cs_cpufreq_notifier_block = {
126 .notifier_call = dbs_cpufreq_notifier,
127};
128
129/************************** sysfs interface ************************/ 118/************************** sysfs interface ************************/
130static struct dbs_governor cs_dbs_gov;
131 119
132static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set, 120static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set,
133 const char *buf, size_t count) 121 const char *buf, size_t count)
@@ -268,15 +256,13 @@ static void cs_free(struct policy_dbs_info *policy_dbs)
268 kfree(to_dbs_info(policy_dbs)); 256 kfree(to_dbs_info(policy_dbs));
269} 257}
270 258
271static int cs_init(struct dbs_data *dbs_data, bool notify) 259static int cs_init(struct dbs_data *dbs_data)
272{ 260{
273 struct cs_dbs_tuners *tuners; 261 struct cs_dbs_tuners *tuners;
274 262
275 tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); 263 tuners = kzalloc(sizeof(*tuners), GFP_KERNEL);
276 if (!tuners) { 264 if (!tuners)
277 pr_err("%s: kzalloc failed\n", __func__);
278 return -ENOMEM; 265 return -ENOMEM;
279 }
280 266
281 tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD; 267 tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD;
282 tuners->freq_step = DEF_FREQUENCY_STEP; 268 tuners->freq_step = DEF_FREQUENCY_STEP;
@@ -288,19 +274,11 @@ static int cs_init(struct dbs_data *dbs_data, bool notify)
288 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * 274 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
289 jiffies_to_usecs(10); 275 jiffies_to_usecs(10);
290 276
291 if (notify)
292 cpufreq_register_notifier(&cs_cpufreq_notifier_block,
293 CPUFREQ_TRANSITION_NOTIFIER);
294
295 return 0; 277 return 0;
296} 278}
297 279
298static void cs_exit(struct dbs_data *dbs_data, bool notify) 280static void cs_exit(struct dbs_data *dbs_data)
299{ 281{
300 if (notify)
301 cpufreq_unregister_notifier(&cs_cpufreq_notifier_block,
302 CPUFREQ_TRANSITION_NOTIFIER);
303
304 kfree(dbs_data->tuners); 282 kfree(dbs_data->tuners);
305} 283}
306 284
@@ -309,16 +287,10 @@ static void cs_start(struct cpufreq_policy *policy)
309 struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); 287 struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
310 288
311 dbs_info->down_skip = 0; 289 dbs_info->down_skip = 0;
312 dbs_info->requested_freq = policy->cur;
313} 290}
314 291
315static struct dbs_governor cs_dbs_gov = { 292static struct dbs_governor cs_governor = {
316 .gov = { 293 .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"),
317 .name = "conservative",
318 .governor = cpufreq_governor_dbs,
319 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
320 .owner = THIS_MODULE,
321 },
322 .kobj_type = { .default_attrs = cs_attributes }, 294 .kobj_type = { .default_attrs = cs_attributes },
323 .gov_dbs_timer = cs_dbs_timer, 295 .gov_dbs_timer = cs_dbs_timer,
324 .alloc = cs_alloc, 296 .alloc = cs_alloc,
@@ -328,33 +300,7 @@ static struct dbs_governor cs_dbs_gov = {
328 .start = cs_start, 300 .start = cs_start,
329}; 301};
330 302
331#define CPU_FREQ_GOV_CONSERVATIVE (&cs_dbs_gov.gov) 303#define CPU_FREQ_GOV_CONSERVATIVE (&cs_governor.gov)
332
333static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
334 void *data)
335{
336 struct cpufreq_freqs *freq = data;
337 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(freq->cpu);
338 struct cs_policy_dbs_info *dbs_info;
339
340 if (!policy)
341 return 0;
342
343 /* policy isn't governed by conservative governor */
344 if (policy->governor != CPU_FREQ_GOV_CONSERVATIVE)
345 return 0;
346
347 dbs_info = to_dbs_info(policy->governor_data);
348 /*
349 * we only care if our internally tracked freq moves outside the 'valid'
350 * ranges of frequency available to us otherwise we do not change it
351 */
352 if (dbs_info->requested_freq > policy->max
353 || dbs_info->requested_freq < policy->min)
354 dbs_info->requested_freq = freq->new;
355
356 return 0;
357}
358 304
359static int __init cpufreq_gov_dbs_init(void) 305static int __init cpufreq_gov_dbs_init(void)
360{ 306{
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index be498d56dd69..e415349ab31b 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -336,17 +336,6 @@ static inline void gov_clear_update_util(struct cpufreq_policy *policy)
336 synchronize_sched(); 336 synchronize_sched();
337} 337}
338 338
339static void gov_cancel_work(struct cpufreq_policy *policy)
340{
341 struct policy_dbs_info *policy_dbs = policy->governor_data;
342
343 gov_clear_update_util(policy_dbs->policy);
344 irq_work_sync(&policy_dbs->irq_work);
345 cancel_work_sync(&policy_dbs->work);
346 atomic_set(&policy_dbs->work_count, 0);
347 policy_dbs->work_in_progress = false;
348}
349
350static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy, 339static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy,
351 struct dbs_governor *gov) 340 struct dbs_governor *gov)
352{ 341{
@@ -389,7 +378,7 @@ static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs,
389 gov->free(policy_dbs); 378 gov->free(policy_dbs);
390} 379}
391 380
392static int cpufreq_governor_init(struct cpufreq_policy *policy) 381int cpufreq_dbs_governor_init(struct cpufreq_policy *policy)
393{ 382{
394 struct dbs_governor *gov = dbs_governor_of(policy); 383 struct dbs_governor *gov = dbs_governor_of(policy);
395 struct dbs_data *dbs_data; 384 struct dbs_data *dbs_data;
@@ -429,7 +418,7 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy)
429 418
430 gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list); 419 gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list);
431 420
432 ret = gov->init(dbs_data, !policy->governor->initialized); 421 ret = gov->init(dbs_data);
433 if (ret) 422 if (ret)
434 goto free_policy_dbs_info; 423 goto free_policy_dbs_info;
435 424
@@ -458,13 +447,13 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy)
458 goto out; 447 goto out;
459 448
460 /* Failure, so roll back. */ 449 /* Failure, so roll back. */
461 pr_err("cpufreq: Governor initialization failed (dbs_data kobject init error %d)\n", ret); 450 pr_err("initialization failed (dbs_data kobject init error %d)\n", ret);
462 451
463 policy->governor_data = NULL; 452 policy->governor_data = NULL;
464 453
465 if (!have_governor_per_policy()) 454 if (!have_governor_per_policy())
466 gov->gdbs_data = NULL; 455 gov->gdbs_data = NULL;
467 gov->exit(dbs_data, !policy->governor->initialized); 456 gov->exit(dbs_data);
468 kfree(dbs_data); 457 kfree(dbs_data);
469 458
470free_policy_dbs_info: 459free_policy_dbs_info:
@@ -474,8 +463,9 @@ out:
474 mutex_unlock(&gov_dbs_data_mutex); 463 mutex_unlock(&gov_dbs_data_mutex);
475 return ret; 464 return ret;
476} 465}
466EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_init);
477 467
478static int cpufreq_governor_exit(struct cpufreq_policy *policy) 468void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy)
479{ 469{
480 struct dbs_governor *gov = dbs_governor_of(policy); 470 struct dbs_governor *gov = dbs_governor_of(policy);
481 struct policy_dbs_info *policy_dbs = policy->governor_data; 471 struct policy_dbs_info *policy_dbs = policy->governor_data;
@@ -493,17 +483,17 @@ static int cpufreq_governor_exit(struct cpufreq_policy *policy)
493 if (!have_governor_per_policy()) 483 if (!have_governor_per_policy())
494 gov->gdbs_data = NULL; 484 gov->gdbs_data = NULL;
495 485
496 gov->exit(dbs_data, policy->governor->initialized == 1); 486 gov->exit(dbs_data);
497 kfree(dbs_data); 487 kfree(dbs_data);
498 } 488 }
499 489
500 free_policy_dbs_info(policy_dbs, gov); 490 free_policy_dbs_info(policy_dbs, gov);
501 491
502 mutex_unlock(&gov_dbs_data_mutex); 492 mutex_unlock(&gov_dbs_data_mutex);
503 return 0;
504} 493}
494EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_exit);
505 495
506static int cpufreq_governor_start(struct cpufreq_policy *policy) 496int cpufreq_dbs_governor_start(struct cpufreq_policy *policy)
507{ 497{
508 struct dbs_governor *gov = dbs_governor_of(policy); 498 struct dbs_governor *gov = dbs_governor_of(policy);
509 struct policy_dbs_info *policy_dbs = policy->governor_data; 499 struct policy_dbs_info *policy_dbs = policy->governor_data;
@@ -539,47 +529,28 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy)
539 gov_set_update_util(policy_dbs, sampling_rate); 529 gov_set_update_util(policy_dbs, sampling_rate);
540 return 0; 530 return 0;
541} 531}
532EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_start);
542 533
543static int cpufreq_governor_stop(struct cpufreq_policy *policy) 534void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy)
544{ 535{
545 gov_cancel_work(policy); 536 struct policy_dbs_info *policy_dbs = policy->governor_data;
546 return 0; 537
538 gov_clear_update_util(policy_dbs->policy);
539 irq_work_sync(&policy_dbs->irq_work);
540 cancel_work_sync(&policy_dbs->work);
541 atomic_set(&policy_dbs->work_count, 0);
542 policy_dbs->work_in_progress = false;
547} 543}
544EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_stop);
548 545
549static int cpufreq_governor_limits(struct cpufreq_policy *policy) 546void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy)
550{ 547{
551 struct policy_dbs_info *policy_dbs = policy->governor_data; 548 struct policy_dbs_info *policy_dbs = policy->governor_data;
552 549
553 mutex_lock(&policy_dbs->timer_mutex); 550 mutex_lock(&policy_dbs->timer_mutex);
554 551 cpufreq_policy_apply_limits(policy);
555 if (policy->max < policy->cur)
556 __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H);
557 else if (policy->min > policy->cur)
558 __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L);
559
560 gov_update_sample_delay(policy_dbs, 0); 552 gov_update_sample_delay(policy_dbs, 0);
561 553
562 mutex_unlock(&policy_dbs->timer_mutex); 554 mutex_unlock(&policy_dbs->timer_mutex);
563
564 return 0;
565}
566
567int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event)
568{
569 if (event == CPUFREQ_GOV_POLICY_INIT) {
570 return cpufreq_governor_init(policy);
571 } else if (policy->governor_data) {
572 switch (event) {
573 case CPUFREQ_GOV_POLICY_EXIT:
574 return cpufreq_governor_exit(policy);
575 case CPUFREQ_GOV_START:
576 return cpufreq_governor_start(policy);
577 case CPUFREQ_GOV_STOP:
578 return cpufreq_governor_stop(policy);
579 case CPUFREQ_GOV_LIMITS:
580 return cpufreq_governor_limits(policy);
581 }
582 }
583 return -EINVAL;
584} 555}
585EXPORT_SYMBOL_GPL(cpufreq_governor_dbs); 556EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_limits);
diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h
index 34eb214b6d57..ef1037e9c92b 100644
--- a/drivers/cpufreq/cpufreq_governor.h
+++ b/drivers/cpufreq/cpufreq_governor.h
@@ -138,8 +138,8 @@ struct dbs_governor {
138 unsigned int (*gov_dbs_timer)(struct cpufreq_policy *policy); 138 unsigned int (*gov_dbs_timer)(struct cpufreq_policy *policy);
139 struct policy_dbs_info *(*alloc)(void); 139 struct policy_dbs_info *(*alloc)(void);
140 void (*free)(struct policy_dbs_info *policy_dbs); 140 void (*free)(struct policy_dbs_info *policy_dbs);
141 int (*init)(struct dbs_data *dbs_data, bool notify); 141 int (*init)(struct dbs_data *dbs_data);
142 void (*exit)(struct dbs_data *dbs_data, bool notify); 142 void (*exit)(struct dbs_data *dbs_data);
143 void (*start)(struct cpufreq_policy *policy); 143 void (*start)(struct cpufreq_policy *policy);
144}; 144};
145 145
@@ -148,6 +148,25 @@ static inline struct dbs_governor *dbs_governor_of(struct cpufreq_policy *policy
148 return container_of(policy->governor, struct dbs_governor, gov); 148 return container_of(policy->governor, struct dbs_governor, gov);
149} 149}
150 150
151/* Governor callback routines */
152int cpufreq_dbs_governor_init(struct cpufreq_policy *policy);
153void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy);
154int cpufreq_dbs_governor_start(struct cpufreq_policy *policy);
155void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy);
156void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy);
157
158#define CPUFREQ_DBS_GOVERNOR_INITIALIZER(_name_) \
159 { \
160 .name = _name_, \
161 .max_transition_latency = TRANSITION_LATENCY_LIMIT, \
162 .owner = THIS_MODULE, \
163 .init = cpufreq_dbs_governor_init, \
164 .exit = cpufreq_dbs_governor_exit, \
165 .start = cpufreq_dbs_governor_start, \
166 .stop = cpufreq_dbs_governor_stop, \
167 .limits = cpufreq_dbs_governor_limits, \
168 }
169
151/* Governor specific operations */ 170/* Governor specific operations */
152struct od_ops { 171struct od_ops {
153 unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy, 172 unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy,
@@ -155,7 +174,6 @@ struct od_ops {
155}; 174};
156 175
157unsigned int dbs_update(struct cpufreq_policy *policy); 176unsigned int dbs_update(struct cpufreq_policy *policy);
158int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event);
159void od_register_powersave_bias_handler(unsigned int (*f) 177void od_register_powersave_bias_handler(unsigned int (*f)
160 (struct cpufreq_policy *, unsigned int, unsigned int), 178 (struct cpufreq_policy *, unsigned int, unsigned int),
161 unsigned int powersave_bias); 179 unsigned int powersave_bias);
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
index 300163430516..3a1f49f5f4c6 100644
--- a/drivers/cpufreq/cpufreq_ondemand.c
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -65,34 +65,30 @@ static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy,
65{ 65{
66 unsigned int freq_req, freq_reduc, freq_avg; 66 unsigned int freq_req, freq_reduc, freq_avg;
67 unsigned int freq_hi, freq_lo; 67 unsigned int freq_hi, freq_lo;
68 unsigned int index = 0; 68 unsigned int index;
69 unsigned int delay_hi_us; 69 unsigned int delay_hi_us;
70 struct policy_dbs_info *policy_dbs = policy->governor_data; 70 struct policy_dbs_info *policy_dbs = policy->governor_data;
71 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); 71 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
72 struct dbs_data *dbs_data = policy_dbs->dbs_data; 72 struct dbs_data *dbs_data = policy_dbs->dbs_data;
73 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 73 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
74 struct cpufreq_frequency_table *freq_table = policy->freq_table;
74 75
75 if (!dbs_info->freq_table) { 76 if (!freq_table) {
76 dbs_info->freq_lo = 0; 77 dbs_info->freq_lo = 0;
77 dbs_info->freq_lo_delay_us = 0; 78 dbs_info->freq_lo_delay_us = 0;
78 return freq_next; 79 return freq_next;
79 } 80 }
80 81
81 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next, 82 index = cpufreq_frequency_table_target(policy, freq_next, relation);
82 relation, &index); 83 freq_req = freq_table[index].frequency;
83 freq_req = dbs_info->freq_table[index].frequency;
84 freq_reduc = freq_req * od_tuners->powersave_bias / 1000; 84 freq_reduc = freq_req * od_tuners->powersave_bias / 1000;
85 freq_avg = freq_req - freq_reduc; 85 freq_avg = freq_req - freq_reduc;
86 86
87 /* Find freq bounds for freq_avg in freq_table */ 87 /* Find freq bounds for freq_avg in freq_table */
88 index = 0; 88 index = cpufreq_table_find_index_h(policy, freq_avg);
89 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg, 89 freq_lo = freq_table[index].frequency;
90 CPUFREQ_RELATION_H, &index); 90 index = cpufreq_table_find_index_l(policy, freq_avg);
91 freq_lo = dbs_info->freq_table[index].frequency; 91 freq_hi = freq_table[index].frequency;
92 index = 0;
93 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
94 CPUFREQ_RELATION_L, &index);
95 freq_hi = dbs_info->freq_table[index].frequency;
96 92
97 /* Find out how long we have to be in hi and lo freqs */ 93 /* Find out how long we have to be in hi and lo freqs */
98 if (freq_hi == freq_lo) { 94 if (freq_hi == freq_lo) {
@@ -113,7 +109,6 @@ static void ondemand_powersave_bias_init(struct cpufreq_policy *policy)
113{ 109{
114 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); 110 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
115 111
116 dbs_info->freq_table = cpufreq_frequency_get_table(policy->cpu);
117 dbs_info->freq_lo = 0; 112 dbs_info->freq_lo = 0;
118} 113}
119 114
@@ -361,17 +356,15 @@ static void od_free(struct policy_dbs_info *policy_dbs)
361 kfree(to_dbs_info(policy_dbs)); 356 kfree(to_dbs_info(policy_dbs));
362} 357}
363 358
364static int od_init(struct dbs_data *dbs_data, bool notify) 359static int od_init(struct dbs_data *dbs_data)
365{ 360{
366 struct od_dbs_tuners *tuners; 361 struct od_dbs_tuners *tuners;
367 u64 idle_time; 362 u64 idle_time;
368 int cpu; 363 int cpu;
369 364
370 tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); 365 tuners = kzalloc(sizeof(*tuners), GFP_KERNEL);
371 if (!tuners) { 366 if (!tuners)
372 pr_err("%s: kzalloc failed\n", __func__);
373 return -ENOMEM; 367 return -ENOMEM;
374 }
375 368
376 cpu = get_cpu(); 369 cpu = get_cpu();
377 idle_time = get_cpu_idle_time_us(cpu, NULL); 370 idle_time = get_cpu_idle_time_us(cpu, NULL);
@@ -402,7 +395,7 @@ static int od_init(struct dbs_data *dbs_data, bool notify)
402 return 0; 395 return 0;
403} 396}
404 397
405static void od_exit(struct dbs_data *dbs_data, bool notify) 398static void od_exit(struct dbs_data *dbs_data)
406{ 399{
407 kfree(dbs_data->tuners); 400 kfree(dbs_data->tuners);
408} 401}
@@ -420,12 +413,7 @@ static struct od_ops od_ops = {
420}; 413};
421 414
422static struct dbs_governor od_dbs_gov = { 415static struct dbs_governor od_dbs_gov = {
423 .gov = { 416 .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"),
424 .name = "ondemand",
425 .governor = cpufreq_governor_dbs,
426 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
427 .owner = THIS_MODULE,
428 },
429 .kobj_type = { .default_attrs = od_attributes }, 417 .kobj_type = { .default_attrs = od_attributes },
430 .gov_dbs_timer = od_dbs_timer, 418 .gov_dbs_timer = od_dbs_timer,
431 .alloc = od_alloc, 419 .alloc = od_alloc,
diff --git a/drivers/cpufreq/cpufreq_ondemand.h b/drivers/cpufreq/cpufreq_ondemand.h
index f0121db3cd9e..640ea4e97106 100644
--- a/drivers/cpufreq/cpufreq_ondemand.h
+++ b/drivers/cpufreq/cpufreq_ondemand.h
@@ -13,7 +13,6 @@
13 13
14struct od_policy_dbs_info { 14struct od_policy_dbs_info {
15 struct policy_dbs_info policy_dbs; 15 struct policy_dbs_info policy_dbs;
16 struct cpufreq_frequency_table *freq_table;
17 unsigned int freq_lo; 16 unsigned int freq_lo;
18 unsigned int freq_lo_delay_us; 17 unsigned int freq_lo_delay_us;
19 unsigned int freq_hi_delay_us; 18 unsigned int freq_hi_delay_us;
diff --git a/drivers/cpufreq/cpufreq_performance.c b/drivers/cpufreq/cpufreq_performance.c
index af9f4b96f5a8..dafb679adc58 100644
--- a/drivers/cpufreq/cpufreq_performance.c
+++ b/drivers/cpufreq/cpufreq_performance.c
@@ -16,27 +16,16 @@
16#include <linux/init.h> 16#include <linux/init.h>
17#include <linux/module.h> 17#include <linux/module.h>
18 18
19static int cpufreq_governor_performance(struct cpufreq_policy *policy, 19static void cpufreq_gov_performance_limits(struct cpufreq_policy *policy)
20 unsigned int event)
21{ 20{
22 switch (event) { 21 pr_debug("setting to %u kHz\n", policy->max);
23 case CPUFREQ_GOV_START: 22 __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H);
24 case CPUFREQ_GOV_LIMITS:
25 pr_debug("setting to %u kHz because of event %u\n",
26 policy->max, event);
27 __cpufreq_driver_target(policy, policy->max,
28 CPUFREQ_RELATION_H);
29 break;
30 default:
31 break;
32 }
33 return 0;
34} 23}
35 24
36static struct cpufreq_governor cpufreq_gov_performance = { 25static struct cpufreq_governor cpufreq_gov_performance = {
37 .name = "performance", 26 .name = "performance",
38 .governor = cpufreq_governor_performance,
39 .owner = THIS_MODULE, 27 .owner = THIS_MODULE,
28 .limits = cpufreq_gov_performance_limits,
40}; 29};
41 30
42static int __init cpufreq_gov_performance_init(void) 31static int __init cpufreq_gov_performance_init(void)
diff --git a/drivers/cpufreq/cpufreq_powersave.c b/drivers/cpufreq/cpufreq_powersave.c
index b8b400232a74..78a651038faf 100644
--- a/drivers/cpufreq/cpufreq_powersave.c
+++ b/drivers/cpufreq/cpufreq_powersave.c
@@ -16,26 +16,15 @@
16#include <linux/init.h> 16#include <linux/init.h>
17#include <linux/module.h> 17#include <linux/module.h>
18 18
19static int cpufreq_governor_powersave(struct cpufreq_policy *policy, 19static void cpufreq_gov_powersave_limits(struct cpufreq_policy *policy)
20 unsigned int event)
21{ 20{
22 switch (event) { 21 pr_debug("setting to %u kHz\n", policy->min);
23 case CPUFREQ_GOV_START: 22 __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L);
24 case CPUFREQ_GOV_LIMITS:
25 pr_debug("setting to %u kHz because of event %u\n",
26 policy->min, event);
27 __cpufreq_driver_target(policy, policy->min,
28 CPUFREQ_RELATION_L);
29 break;
30 default:
31 break;
32 }
33 return 0;
34} 23}
35 24
36static struct cpufreq_governor cpufreq_gov_powersave = { 25static struct cpufreq_governor cpufreq_gov_powersave = {
37 .name = "powersave", 26 .name = "powersave",
38 .governor = cpufreq_governor_powersave, 27 .limits = cpufreq_gov_powersave_limits,
39 .owner = THIS_MODULE, 28 .owner = THIS_MODULE,
40}; 29};
41 30
diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c
index 5e370a30a964..06d3abdffd3a 100644
--- a/drivers/cpufreq/cpufreq_stats.c
+++ b/drivers/cpufreq/cpufreq_stats.c
@@ -15,7 +15,7 @@
15#include <linux/slab.h> 15#include <linux/slab.h>
16#include <linux/cputime.h> 16#include <linux/cputime.h>
17 17
18static spinlock_t cpufreq_stats_lock; 18static DEFINE_SPINLOCK(cpufreq_stats_lock);
19 19
20struct cpufreq_stats { 20struct cpufreq_stats {
21 unsigned int total_trans; 21 unsigned int total_trans;
@@ -52,6 +52,9 @@ static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf)
52 ssize_t len = 0; 52 ssize_t len = 0;
53 int i; 53 int i;
54 54
55 if (policy->fast_switch_enabled)
56 return 0;
57
55 cpufreq_stats_update(stats); 58 cpufreq_stats_update(stats);
56 for (i = 0; i < stats->state_num; i++) { 59 for (i = 0; i < stats->state_num; i++) {
57 len += sprintf(buf + len, "%u %llu\n", stats->freq_table[i], 60 len += sprintf(buf + len, "%u %llu\n", stats->freq_table[i],
@@ -68,6 +71,9 @@ static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf)
68 ssize_t len = 0; 71 ssize_t len = 0;
69 int i, j; 72 int i, j;
70 73
74 if (policy->fast_switch_enabled)
75 return 0;
76
71 len += snprintf(buf + len, PAGE_SIZE - len, " From : To\n"); 77 len += snprintf(buf + len, PAGE_SIZE - len, " From : To\n");
72 len += snprintf(buf + len, PAGE_SIZE - len, " : "); 78 len += snprintf(buf + len, PAGE_SIZE - len, " : ");
73 for (i = 0; i < stats->state_num; i++) { 79 for (i = 0; i < stats->state_num; i++) {
@@ -130,7 +136,7 @@ static int freq_table_get_index(struct cpufreq_stats *stats, unsigned int freq)
130 return -1; 136 return -1;
131} 137}
132 138
133static void __cpufreq_stats_free_table(struct cpufreq_policy *policy) 139void cpufreq_stats_free_table(struct cpufreq_policy *policy)
134{ 140{
135 struct cpufreq_stats *stats = policy->stats; 141 struct cpufreq_stats *stats = policy->stats;
136 142
@@ -146,39 +152,25 @@ static void __cpufreq_stats_free_table(struct cpufreq_policy *policy)
146 policy->stats = NULL; 152 policy->stats = NULL;
147} 153}
148 154
149static void cpufreq_stats_free_table(unsigned int cpu) 155void cpufreq_stats_create_table(struct cpufreq_policy *policy)
150{
151 struct cpufreq_policy *policy;
152
153 policy = cpufreq_cpu_get(cpu);
154 if (!policy)
155 return;
156
157 __cpufreq_stats_free_table(policy);
158
159 cpufreq_cpu_put(policy);
160}
161
162static int __cpufreq_stats_create_table(struct cpufreq_policy *policy)
163{ 156{
164 unsigned int i = 0, count = 0, ret = -ENOMEM; 157 unsigned int i = 0, count = 0, ret = -ENOMEM;
165 struct cpufreq_stats *stats; 158 struct cpufreq_stats *stats;
166 unsigned int alloc_size; 159 unsigned int alloc_size;
167 unsigned int cpu = policy->cpu;
168 struct cpufreq_frequency_table *pos, *table; 160 struct cpufreq_frequency_table *pos, *table;
169 161
170 /* We need cpufreq table for creating stats table */ 162 /* We need cpufreq table for creating stats table */
171 table = cpufreq_frequency_get_table(cpu); 163 table = policy->freq_table;
172 if (unlikely(!table)) 164 if (unlikely(!table))
173 return 0; 165 return;
174 166
175 /* stats already initialized */ 167 /* stats already initialized */
176 if (policy->stats) 168 if (policy->stats)
177 return -EEXIST; 169 return;
178 170
179 stats = kzalloc(sizeof(*stats), GFP_KERNEL); 171 stats = kzalloc(sizeof(*stats), GFP_KERNEL);
180 if (!stats) 172 if (!stats)
181 return -ENOMEM; 173 return;
182 174
183 /* Find total allocation size */ 175 /* Find total allocation size */
184 cpufreq_for_each_valid_entry(pos, table) 176 cpufreq_for_each_valid_entry(pos, table)
@@ -215,80 +207,32 @@ static int __cpufreq_stats_create_table(struct cpufreq_policy *policy)
215 policy->stats = stats; 207 policy->stats = stats;
216 ret = sysfs_create_group(&policy->kobj, &stats_attr_group); 208 ret = sysfs_create_group(&policy->kobj, &stats_attr_group);
217 if (!ret) 209 if (!ret)
218 return 0; 210 return;
219 211
220 /* We failed, release resources */ 212 /* We failed, release resources */
221 policy->stats = NULL; 213 policy->stats = NULL;
222 kfree(stats->time_in_state); 214 kfree(stats->time_in_state);
223free_stat: 215free_stat:
224 kfree(stats); 216 kfree(stats);
225
226 return ret;
227}
228
229static void cpufreq_stats_create_table(unsigned int cpu)
230{
231 struct cpufreq_policy *policy;
232
233 /*
234 * "likely(!policy)" because normally cpufreq_stats will be registered
235 * before cpufreq driver
236 */
237 policy = cpufreq_cpu_get(cpu);
238 if (likely(!policy))
239 return;
240
241 __cpufreq_stats_create_table(policy);
242
243 cpufreq_cpu_put(policy);
244} 217}
245 218
246static int cpufreq_stat_notifier_policy(struct notifier_block *nb, 219void cpufreq_stats_record_transition(struct cpufreq_policy *policy,
247 unsigned long val, void *data) 220 unsigned int new_freq)
248{ 221{
249 int ret = 0; 222 struct cpufreq_stats *stats = policy->stats;
250 struct cpufreq_policy *policy = data;
251
252 if (val == CPUFREQ_CREATE_POLICY)
253 ret = __cpufreq_stats_create_table(policy);
254 else if (val == CPUFREQ_REMOVE_POLICY)
255 __cpufreq_stats_free_table(policy);
256
257 return ret;
258}
259
260static int cpufreq_stat_notifier_trans(struct notifier_block *nb,
261 unsigned long val, void *data)
262{
263 struct cpufreq_freqs *freq = data;
264 struct cpufreq_policy *policy = cpufreq_cpu_get(freq->cpu);
265 struct cpufreq_stats *stats;
266 int old_index, new_index; 223 int old_index, new_index;
267 224
268 if (!policy) { 225 if (!stats) {
269 pr_err("%s: No policy found\n", __func__);
270 return 0;
271 }
272
273 if (val != CPUFREQ_POSTCHANGE)
274 goto put_policy;
275
276 if (!policy->stats) {
277 pr_debug("%s: No stats found\n", __func__); 226 pr_debug("%s: No stats found\n", __func__);
278 goto put_policy; 227 return;
279 } 228 }
280 229
281 stats = policy->stats;
282
283 old_index = stats->last_index; 230 old_index = stats->last_index;
284 new_index = freq_table_get_index(stats, freq->new); 231 new_index = freq_table_get_index(stats, new_freq);
285 232
286 /* We can't do stats->time_in_state[-1]= .. */ 233 /* We can't do stats->time_in_state[-1]= .. */
287 if (old_index == -1 || new_index == -1) 234 if (old_index == -1 || new_index == -1 || old_index == new_index)
288 goto put_policy; 235 return;
289
290 if (old_index == new_index)
291 goto put_policy;
292 236
293 cpufreq_stats_update(stats); 237 cpufreq_stats_update(stats);
294 238
@@ -297,61 +241,4 @@ static int cpufreq_stat_notifier_trans(struct notifier_block *nb,
297 stats->trans_table[old_index * stats->max_state + new_index]++; 241 stats->trans_table[old_index * stats->max_state + new_index]++;
298#endif 242#endif
299 stats->total_trans++; 243 stats->total_trans++;
300
301put_policy:
302 cpufreq_cpu_put(policy);
303 return 0;
304} 244}
305
306static struct notifier_block notifier_policy_block = {
307 .notifier_call = cpufreq_stat_notifier_policy
308};
309
310static struct notifier_block notifier_trans_block = {
311 .notifier_call = cpufreq_stat_notifier_trans
312};
313
314static int __init cpufreq_stats_init(void)
315{
316 int ret;
317 unsigned int cpu;
318
319 spin_lock_init(&cpufreq_stats_lock);
320 ret = cpufreq_register_notifier(&notifier_policy_block,
321 CPUFREQ_POLICY_NOTIFIER);
322 if (ret)
323 return ret;
324
325 for_each_online_cpu(cpu)
326 cpufreq_stats_create_table(cpu);
327
328 ret = cpufreq_register_notifier(&notifier_trans_block,
329 CPUFREQ_TRANSITION_NOTIFIER);
330 if (ret) {
331 cpufreq_unregister_notifier(&notifier_policy_block,
332 CPUFREQ_POLICY_NOTIFIER);
333 for_each_online_cpu(cpu)
334 cpufreq_stats_free_table(cpu);
335 return ret;
336 }
337
338 return 0;
339}
340static void __exit cpufreq_stats_exit(void)
341{
342 unsigned int cpu;
343
344 cpufreq_unregister_notifier(&notifier_policy_block,
345 CPUFREQ_POLICY_NOTIFIER);
346 cpufreq_unregister_notifier(&notifier_trans_block,
347 CPUFREQ_TRANSITION_NOTIFIER);
348 for_each_online_cpu(cpu)
349 cpufreq_stats_free_table(cpu);
350}
351
352MODULE_AUTHOR("Zou Nan hai <nanhai.zou@intel.com>");
353MODULE_DESCRIPTION("Export cpufreq stats via sysfs");
354MODULE_LICENSE("GPL");
355
356module_init(cpufreq_stats_init);
357module_exit(cpufreq_stats_exit);
diff --git a/drivers/cpufreq/cpufreq_userspace.c b/drivers/cpufreq/cpufreq_userspace.c
index 9f3dec9a3f36..bd897e3e134d 100644
--- a/drivers/cpufreq/cpufreq_userspace.c
+++ b/drivers/cpufreq/cpufreq_userspace.c
@@ -65,66 +65,66 @@ static int cpufreq_userspace_policy_init(struct cpufreq_policy *policy)
65 return 0; 65 return 0;
66} 66}
67 67
68static int cpufreq_governor_userspace(struct cpufreq_policy *policy, 68static void cpufreq_userspace_policy_exit(struct cpufreq_policy *policy)
69 unsigned int event) 69{
70 mutex_lock(&userspace_mutex);
71 kfree(policy->governor_data);
72 policy->governor_data = NULL;
73 mutex_unlock(&userspace_mutex);
74}
75
76static int cpufreq_userspace_policy_start(struct cpufreq_policy *policy)
70{ 77{
71 unsigned int *setspeed = policy->governor_data; 78 unsigned int *setspeed = policy->governor_data;
72 unsigned int cpu = policy->cpu;
73 int rc = 0;
74 79
75 if (event == CPUFREQ_GOV_POLICY_INIT) 80 BUG_ON(!policy->cur);
76 return cpufreq_userspace_policy_init(policy); 81 pr_debug("started managing cpu %u\n", policy->cpu);
77 82
78 if (!setspeed) 83 mutex_lock(&userspace_mutex);
79 return -EINVAL; 84 per_cpu(cpu_is_managed, policy->cpu) = 1;
80 85 *setspeed = policy->cur;
81 switch (event) { 86 mutex_unlock(&userspace_mutex);
82 case CPUFREQ_GOV_POLICY_EXIT: 87 return 0;
83 mutex_lock(&userspace_mutex); 88}
84 policy->governor_data = NULL; 89
85 kfree(setspeed); 90static void cpufreq_userspace_policy_stop(struct cpufreq_policy *policy)
86 mutex_unlock(&userspace_mutex); 91{
87 break; 92 unsigned int *setspeed = policy->governor_data;
88 case CPUFREQ_GOV_START: 93
89 BUG_ON(!policy->cur); 94 pr_debug("managing cpu %u stopped\n", policy->cpu);
90 pr_debug("started managing cpu %u\n", cpu); 95
91 96 mutex_lock(&userspace_mutex);
92 mutex_lock(&userspace_mutex); 97 per_cpu(cpu_is_managed, policy->cpu) = 0;
93 per_cpu(cpu_is_managed, cpu) = 1; 98 *setspeed = 0;
94 *setspeed = policy->cur; 99 mutex_unlock(&userspace_mutex);
95 mutex_unlock(&userspace_mutex); 100}
96 break; 101
97 case CPUFREQ_GOV_STOP: 102static void cpufreq_userspace_policy_limits(struct cpufreq_policy *policy)
98 pr_debug("managing cpu %u stopped\n", cpu); 103{
99 104 unsigned int *setspeed = policy->governor_data;
100 mutex_lock(&userspace_mutex); 105
101 per_cpu(cpu_is_managed, cpu) = 0; 106 mutex_lock(&userspace_mutex);
102 *setspeed = 0; 107
103 mutex_unlock(&userspace_mutex); 108 pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz, last set to %u kHz\n",
104 break; 109 policy->cpu, policy->min, policy->max, policy->cur, *setspeed);
105 case CPUFREQ_GOV_LIMITS: 110
106 mutex_lock(&userspace_mutex); 111 if (policy->max < *setspeed)
107 pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz, last set to %u kHz\n", 112 __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H);
108 cpu, policy->min, policy->max, policy->cur, *setspeed); 113 else if (policy->min > *setspeed)
109 114 __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L);
110 if (policy->max < *setspeed) 115 else
111 __cpufreq_driver_target(policy, policy->max, 116 __cpufreq_driver_target(policy, *setspeed, CPUFREQ_RELATION_L);
112 CPUFREQ_RELATION_H); 117
113 else if (policy->min > *setspeed) 118 mutex_unlock(&userspace_mutex);
114 __cpufreq_driver_target(policy, policy->min,
115 CPUFREQ_RELATION_L);
116 else
117 __cpufreq_driver_target(policy, *setspeed,
118 CPUFREQ_RELATION_L);
119 mutex_unlock(&userspace_mutex);
120 break;
121 }
122 return rc;
123} 119}
124 120
125static struct cpufreq_governor cpufreq_gov_userspace = { 121static struct cpufreq_governor cpufreq_gov_userspace = {
126 .name = "userspace", 122 .name = "userspace",
127 .governor = cpufreq_governor_userspace, 123 .init = cpufreq_userspace_policy_init,
124 .exit = cpufreq_userspace_policy_exit,
125 .start = cpufreq_userspace_policy_start,
126 .stop = cpufreq_userspace_policy_stop,
127 .limits = cpufreq_userspace_policy_limits,
128 .store_setspeed = cpufreq_set, 128 .store_setspeed = cpufreq_set,
129 .show_setspeed = show_speed, 129 .show_setspeed = show_speed,
130 .owner = THIS_MODULE, 130 .owner = THIS_MODULE,
diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c
index 7e336d20c184..b95a872800ec 100644
--- a/drivers/cpufreq/davinci-cpufreq.c
+++ b/drivers/cpufreq/davinci-cpufreq.c
@@ -38,26 +38,6 @@ struct davinci_cpufreq {
38}; 38};
39static struct davinci_cpufreq cpufreq; 39static struct davinci_cpufreq cpufreq;
40 40
41static int davinci_verify_speed(struct cpufreq_policy *policy)
42{
43 struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data;
44 struct cpufreq_frequency_table *freq_table = pdata->freq_table;
45 struct clk *armclk = cpufreq.armclk;
46
47 if (freq_table)
48 return cpufreq_frequency_table_verify(policy, freq_table);
49
50 if (policy->cpu)
51 return -EINVAL;
52
53 cpufreq_verify_within_cpu_limits(policy);
54 policy->min = clk_round_rate(armclk, policy->min * 1000) / 1000;
55 policy->max = clk_round_rate(armclk, policy->max * 1000) / 1000;
56 cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
57 policy->cpuinfo.max_freq);
58 return 0;
59}
60
61static int davinci_target(struct cpufreq_policy *policy, unsigned int idx) 41static int davinci_target(struct cpufreq_policy *policy, unsigned int idx)
62{ 42{
63 struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data; 43 struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data;
@@ -121,7 +101,7 @@ static int davinci_cpu_init(struct cpufreq_policy *policy)
121 101
122static struct cpufreq_driver davinci_driver = { 102static struct cpufreq_driver davinci_driver = {
123 .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, 103 .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK,
124 .verify = davinci_verify_speed, 104 .verify = cpufreq_generic_frequency_table_verify,
125 .target_index = davinci_target, 105 .target_index = davinci_target,
126 .get = cpufreq_generic_get, 106 .get = cpufreq_generic_get,
127 .init = davinci_cpu_init, 107 .init = davinci_cpu_init,
diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c
index a8f1daffc9bc..3bbbf9e6960c 100644
--- a/drivers/cpufreq/freq_table.c
+++ b/drivers/cpufreq/freq_table.c
@@ -63,8 +63,6 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy,
63 else 63 else
64 return 0; 64 return 0;
65} 65}
66EXPORT_SYMBOL_GPL(cpufreq_frequency_table_cpuinfo);
67
68 66
69int cpufreq_frequency_table_verify(struct cpufreq_policy *policy, 67int cpufreq_frequency_table_verify(struct cpufreq_policy *policy,
70 struct cpufreq_frequency_table *table) 68 struct cpufreq_frequency_table *table)
@@ -108,20 +106,16 @@ EXPORT_SYMBOL_GPL(cpufreq_frequency_table_verify);
108 */ 106 */
109int cpufreq_generic_frequency_table_verify(struct cpufreq_policy *policy) 107int cpufreq_generic_frequency_table_verify(struct cpufreq_policy *policy)
110{ 108{
111 struct cpufreq_frequency_table *table = 109 if (!policy->freq_table)
112 cpufreq_frequency_get_table(policy->cpu);
113 if (!table)
114 return -ENODEV; 110 return -ENODEV;
115 111
116 return cpufreq_frequency_table_verify(policy, table); 112 return cpufreq_frequency_table_verify(policy, policy->freq_table);
117} 113}
118EXPORT_SYMBOL_GPL(cpufreq_generic_frequency_table_verify); 114EXPORT_SYMBOL_GPL(cpufreq_generic_frequency_table_verify);
119 115
120int cpufreq_frequency_table_target(struct cpufreq_policy *policy, 116int cpufreq_table_index_unsorted(struct cpufreq_policy *policy,
121 struct cpufreq_frequency_table *table, 117 unsigned int target_freq,
122 unsigned int target_freq, 118 unsigned int relation)
123 unsigned int relation,
124 unsigned int *index)
125{ 119{
126 struct cpufreq_frequency_table optimal = { 120 struct cpufreq_frequency_table optimal = {
127 .driver_data = ~0, 121 .driver_data = ~0,
@@ -132,7 +126,9 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy,
132 .frequency = 0, 126 .frequency = 0,
133 }; 127 };
134 struct cpufreq_frequency_table *pos; 128 struct cpufreq_frequency_table *pos;
129 struct cpufreq_frequency_table *table = policy->freq_table;
135 unsigned int freq, diff, i = 0; 130 unsigned int freq, diff, i = 0;
131 int index;
136 132
137 pr_debug("request for target %u kHz (relation: %u) for cpu %u\n", 133 pr_debug("request for target %u kHz (relation: %u) for cpu %u\n",
138 target_freq, relation, policy->cpu); 134 target_freq, relation, policy->cpu);
@@ -196,25 +192,26 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy,
196 } 192 }
197 } 193 }
198 if (optimal.driver_data > i) { 194 if (optimal.driver_data > i) {
199 if (suboptimal.driver_data > i) 195 if (suboptimal.driver_data > i) {
200 return -EINVAL; 196 WARN(1, "Invalid frequency table: %d\n", policy->cpu);
201 *index = suboptimal.driver_data; 197 return 0;
202 } else 198 }
203 *index = optimal.driver_data;
204 199
205 pr_debug("target index is %u, freq is:%u kHz\n", *index, 200 index = suboptimal.driver_data;
206 table[*index].frequency); 201 } else
202 index = optimal.driver_data;
207 203
208 return 0; 204 pr_debug("target index is %u, freq is:%u kHz\n", index,
205 table[index].frequency);
206 return index;
209} 207}
210EXPORT_SYMBOL_GPL(cpufreq_frequency_table_target); 208EXPORT_SYMBOL_GPL(cpufreq_table_index_unsorted);
211 209
212int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy, 210int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy,
213 unsigned int freq) 211 unsigned int freq)
214{ 212{
215 struct cpufreq_frequency_table *pos, *table; 213 struct cpufreq_frequency_table *pos, *table = policy->freq_table;
216 214
217 table = cpufreq_frequency_get_table(policy->cpu);
218 if (unlikely(!table)) { 215 if (unlikely(!table)) {
219 pr_debug("%s: Unable to find frequency table\n", __func__); 216 pr_debug("%s: Unable to find frequency table\n", __func__);
220 return -ENOENT; 217 return -ENOENT;
@@ -300,15 +297,72 @@ struct freq_attr *cpufreq_generic_attr[] = {
300}; 297};
301EXPORT_SYMBOL_GPL(cpufreq_generic_attr); 298EXPORT_SYMBOL_GPL(cpufreq_generic_attr);
302 299
300static int set_freq_table_sorted(struct cpufreq_policy *policy)
301{
302 struct cpufreq_frequency_table *pos, *table = policy->freq_table;
303 struct cpufreq_frequency_table *prev = NULL;
304 int ascending = 0;
305
306 policy->freq_table_sorted = CPUFREQ_TABLE_UNSORTED;
307
308 cpufreq_for_each_valid_entry(pos, table) {
309 if (!prev) {
310 prev = pos;
311 continue;
312 }
313
314 if (pos->frequency == prev->frequency) {
315 pr_warn("Duplicate freq-table entries: %u\n",
316 pos->frequency);
317 return -EINVAL;
318 }
319
320 /* Frequency increased from prev to pos */
321 if (pos->frequency > prev->frequency) {
322 /* But frequency was decreasing earlier */
323 if (ascending < 0) {
324 pr_debug("Freq table is unsorted\n");
325 return 0;
326 }
327
328 ascending++;
329 } else {
330 /* Frequency decreased from prev to pos */
331
332 /* But frequency was increasing earlier */
333 if (ascending > 0) {
334 pr_debug("Freq table is unsorted\n");
335 return 0;
336 }
337
338 ascending--;
339 }
340
341 prev = pos;
342 }
343
344 if (ascending > 0)
345 policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_ASCENDING;
346 else
347 policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_DESCENDING;
348
349 pr_debug("Freq table is sorted in %s order\n",
350 ascending > 0 ? "ascending" : "descending");
351
352 return 0;
353}
354
303int cpufreq_table_validate_and_show(struct cpufreq_policy *policy, 355int cpufreq_table_validate_and_show(struct cpufreq_policy *policy,
304 struct cpufreq_frequency_table *table) 356 struct cpufreq_frequency_table *table)
305{ 357{
306 int ret = cpufreq_frequency_table_cpuinfo(policy, table); 358 int ret;
307 359
308 if (!ret) 360 ret = cpufreq_frequency_table_cpuinfo(policy, table);
309 policy->freq_table = table; 361 if (ret)
362 return ret;
310 363
311 return ret; 364 policy->freq_table = table;
365 return set_freq_table_sorted(policy);
312} 366}
313EXPORT_SYMBOL_GPL(cpufreq_table_validate_and_show); 367EXPORT_SYMBOL_GPL(cpufreq_table_validate_and_show);
314 368
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 28690b284846..d8028def199d 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -97,7 +97,6 @@ static inline u64 div_ext_fp(u64 x, u64 y)
97 * read from MPERF MSR between last and current sample 97 * read from MPERF MSR between last and current sample
98 * @tsc: Difference of time stamp counter between last and 98 * @tsc: Difference of time stamp counter between last and
99 * current sample 99 * current sample
100 * @freq: Effective frequency calculated from APERF/MPERF
101 * @time: Current time from scheduler 100 * @time: Current time from scheduler
102 * 101 *
103 * This structure is used in the cpudata structure to store performance sample 102 * This structure is used in the cpudata structure to store performance sample
@@ -109,7 +108,6 @@ struct sample {
109 u64 aperf; 108 u64 aperf;
110 u64 mperf; 109 u64 mperf;
111 u64 tsc; 110 u64 tsc;
112 int freq;
113 u64 time; 111 u64 time;
114}; 112};
115 113
@@ -282,9 +280,9 @@ struct cpu_defaults {
282static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu); 280static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu);
283static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu); 281static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu);
284 282
285static struct pstate_adjust_policy pid_params; 283static struct pstate_adjust_policy pid_params __read_mostly;
286static struct pstate_funcs pstate_funcs; 284static struct pstate_funcs pstate_funcs __read_mostly;
287static int hwp_active; 285static int hwp_active __read_mostly;
288 286
289#ifdef CONFIG_ACPI 287#ifdef CONFIG_ACPI
290static bool acpi_ppc; 288static bool acpi_ppc;
@@ -808,7 +806,8 @@ static void __init intel_pstate_sysfs_expose_params(void)
808static void intel_pstate_hwp_enable(struct cpudata *cpudata) 806static void intel_pstate_hwp_enable(struct cpudata *cpudata)
809{ 807{
810 /* First disable HWP notification interrupt as we don't process them */ 808 /* First disable HWP notification interrupt as we don't process them */
811 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); 809 if (static_cpu_has(X86_FEATURE_HWP_NOTIFY))
810 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
812 811
813 wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); 812 wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
814} 813}
@@ -945,7 +944,7 @@ static int core_get_max_pstate(void)
945 if (err) 944 if (err)
946 goto skip_tar; 945 goto skip_tar;
947 946
948 tdp_msr = MSR_CONFIG_TDP_NOMINAL + tdp_ctrl; 947 tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x3);
949 err = rdmsrl_safe(tdp_msr, &tdp_ratio); 948 err = rdmsrl_safe(tdp_msr, &tdp_ratio);
950 if (err) 949 if (err)
951 goto skip_tar; 950 goto skip_tar;
@@ -1092,6 +1091,26 @@ static struct cpu_defaults knl_params = {
1092 }, 1091 },
1093}; 1092};
1094 1093
1094static struct cpu_defaults bxt_params = {
1095 .pid_policy = {
1096 .sample_rate_ms = 10,
1097 .deadband = 0,
1098 .setpoint = 60,
1099 .p_gain_pct = 14,
1100 .d_gain_pct = 0,
1101 .i_gain_pct = 4,
1102 },
1103 .funcs = {
1104 .get_max = core_get_max_pstate,
1105 .get_max_physical = core_get_max_pstate_physical,
1106 .get_min = core_get_min_pstate,
1107 .get_turbo = core_get_turbo_pstate,
1108 .get_scaling = core_get_scaling,
1109 .get_val = core_get_val,
1110 .get_target_pstate = get_target_pstate_use_cpu_load,
1111 },
1112};
1113
1095static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) 1114static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
1096{ 1115{
1097 int max_perf = cpu->pstate.turbo_pstate; 1116 int max_perf = cpu->pstate.turbo_pstate;
@@ -1114,17 +1133,12 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
1114 *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf); 1133 *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
1115} 1134}
1116 1135
1117static inline void intel_pstate_record_pstate(struct cpudata *cpu, int pstate)
1118{
1119 trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
1120 cpu->pstate.current_pstate = pstate;
1121}
1122
1123static void intel_pstate_set_min_pstate(struct cpudata *cpu) 1136static void intel_pstate_set_min_pstate(struct cpudata *cpu)
1124{ 1137{
1125 int pstate = cpu->pstate.min_pstate; 1138 int pstate = cpu->pstate.min_pstate;
1126 1139
1127 intel_pstate_record_pstate(cpu, pstate); 1140 trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
1141 cpu->pstate.current_pstate = pstate;
1128 /* 1142 /*
1129 * Generally, there is no guarantee that this code will always run on 1143 * Generally, there is no guarantee that this code will always run on
1130 * the CPU being updated, so force the register update to run on the 1144 * the CPU being updated, so force the register update to run on the
@@ -1284,10 +1298,11 @@ static inline void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
1284 1298
1285 intel_pstate_get_min_max(cpu, &min_perf, &max_perf); 1299 intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
1286 pstate = clamp_t(int, pstate, min_perf, max_perf); 1300 pstate = clamp_t(int, pstate, min_perf, max_perf);
1301 trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
1287 if (pstate == cpu->pstate.current_pstate) 1302 if (pstate == cpu->pstate.current_pstate)
1288 return; 1303 return;
1289 1304
1290 intel_pstate_record_pstate(cpu, pstate); 1305 cpu->pstate.current_pstate = pstate;
1291 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); 1306 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
1292} 1307}
1293 1308
@@ -1352,11 +1367,12 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1352 ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params), 1367 ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params),
1353 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), 1368 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params),
1354 ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params), 1369 ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params),
1370 ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_params),
1355 {} 1371 {}
1356}; 1372};
1357MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); 1373MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
1358 1374
1359static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] = { 1375static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
1360 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), 1376 ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params),
1361 {} 1377 {}
1362}; 1378};
@@ -1576,12 +1592,12 @@ static struct cpufreq_driver intel_pstate_driver = {
1576 .name = "intel_pstate", 1592 .name = "intel_pstate",
1577}; 1593};
1578 1594
1579static int __initdata no_load; 1595static int no_load __initdata;
1580static int __initdata no_hwp; 1596static int no_hwp __initdata;
1581static int __initdata hwp_only; 1597static int hwp_only __initdata;
1582static unsigned int force_load; 1598static unsigned int force_load __initdata;
1583 1599
1584static int intel_pstate_msrs_not_valid(void) 1600static int __init intel_pstate_msrs_not_valid(void)
1585{ 1601{
1586 if (!pstate_funcs.get_max() || 1602 if (!pstate_funcs.get_max() ||
1587 !pstate_funcs.get_min() || 1603 !pstate_funcs.get_min() ||
@@ -1591,7 +1607,7 @@ static int intel_pstate_msrs_not_valid(void)
1591 return 0; 1607 return 0;
1592} 1608}
1593 1609
1594static void copy_pid_params(struct pstate_adjust_policy *policy) 1610static void __init copy_pid_params(struct pstate_adjust_policy *policy)
1595{ 1611{
1596 pid_params.sample_rate_ms = policy->sample_rate_ms; 1612 pid_params.sample_rate_ms = policy->sample_rate_ms;
1597 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; 1613 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
@@ -1602,7 +1618,7 @@ static void copy_pid_params(struct pstate_adjust_policy *policy)
1602 pid_params.setpoint = policy->setpoint; 1618 pid_params.setpoint = policy->setpoint;
1603} 1619}
1604 1620
1605static void copy_cpu_funcs(struct pstate_funcs *funcs) 1621static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
1606{ 1622{
1607 pstate_funcs.get_max = funcs->get_max; 1623 pstate_funcs.get_max = funcs->get_max;
1608 pstate_funcs.get_max_physical = funcs->get_max_physical; 1624 pstate_funcs.get_max_physical = funcs->get_max_physical;
@@ -1617,7 +1633,7 @@ static void copy_cpu_funcs(struct pstate_funcs *funcs)
1617 1633
1618#ifdef CONFIG_ACPI 1634#ifdef CONFIG_ACPI
1619 1635
1620static bool intel_pstate_no_acpi_pss(void) 1636static bool __init intel_pstate_no_acpi_pss(void)
1621{ 1637{
1622 int i; 1638 int i;
1623 1639
@@ -1646,7 +1662,7 @@ static bool intel_pstate_no_acpi_pss(void)
1646 return true; 1662 return true;
1647} 1663}
1648 1664
1649static bool intel_pstate_has_acpi_ppc(void) 1665static bool __init intel_pstate_has_acpi_ppc(void)
1650{ 1666{
1651 int i; 1667 int i;
1652 1668
@@ -1674,7 +1690,7 @@ struct hw_vendor_info {
1674}; 1690};
1675 1691
1676/* Hardware vendor-specific info that has its own power management modes */ 1692/* Hardware vendor-specific info that has its own power management modes */
1677static struct hw_vendor_info vendor_info[] = { 1693static struct hw_vendor_info vendor_info[] __initdata = {
1678 {1, "HP ", "ProLiant", PSS}, 1694 {1, "HP ", "ProLiant", PSS},
1679 {1, "ORACLE", "X4-2 ", PPC}, 1695 {1, "ORACLE", "X4-2 ", PPC},
1680 {1, "ORACLE", "X4-2L ", PPC}, 1696 {1, "ORACLE", "X4-2L ", PPC},
@@ -1693,7 +1709,7 @@ static struct hw_vendor_info vendor_info[] = {
1693 {0, "", ""}, 1709 {0, "", ""},
1694}; 1710};
1695 1711
1696static bool intel_pstate_platform_pwr_mgmt_exists(void) 1712static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
1697{ 1713{
1698 struct acpi_table_header hdr; 1714 struct acpi_table_header hdr;
1699 struct hw_vendor_info *v_info; 1715 struct hw_vendor_info *v_info;
diff --git a/drivers/cpufreq/mvebu-cpufreq.c b/drivers/cpufreq/mvebu-cpufreq.c
index e920889b9ac2..ed915ee85dd9 100644
--- a/drivers/cpufreq/mvebu-cpufreq.c
+++ b/drivers/cpufreq/mvebu-cpufreq.c
@@ -70,7 +70,7 @@ static int __init armada_xp_pmsu_cpufreq_init(void)
70 continue; 70 continue;
71 } 71 }
72 72
73 clk = clk_get(cpu_dev, 0); 73 clk = clk_get(cpu_dev, NULL);
74 if (IS_ERR(clk)) { 74 if (IS_ERR(clk)) {
75 pr_err("Cannot get clock for CPU %d\n", cpu); 75 pr_err("Cannot get clock for CPU %d\n", cpu);
76 return PTR_ERR(clk); 76 return PTR_ERR(clk);
diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c
index a7ecb9a84c15..3f0ce2ae35ee 100644
--- a/drivers/cpufreq/pcc-cpufreq.c
+++ b/drivers/cpufreq/pcc-cpufreq.c
@@ -555,8 +555,6 @@ static int pcc_cpufreq_cpu_init(struct cpufreq_policy *policy)
555 policy->min = policy->cpuinfo.min_freq = 555 policy->min = policy->cpuinfo.min_freq =
556 ioread32(&pcch_hdr->minimum_frequency) * 1000; 556 ioread32(&pcch_hdr->minimum_frequency) * 1000;
557 557
558 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
559
560 pr_debug("init: policy->max is %d, policy->min is %d\n", 558 pr_debug("init: policy->max is %d, policy->min is %d\n",
561 policy->max, policy->min); 559 policy->max, policy->min);
562out: 560out:
diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c
index 54c45368e3f1..0f138b050e9a 100644
--- a/drivers/cpufreq/powernv-cpufreq.c
+++ b/drivers/cpufreq/powernv-cpufreq.c
@@ -64,12 +64,14 @@
64/** 64/**
65 * struct global_pstate_info - Per policy data structure to maintain history of 65 * struct global_pstate_info - Per policy data structure to maintain history of
66 * global pstates 66 * global pstates
67 * @highest_lpstate: The local pstate from which we are ramping down 67 * @highest_lpstate_idx: The local pstate index from which we are
68 * ramping down
68 * @elapsed_time: Time in ms spent in ramping down from 69 * @elapsed_time: Time in ms spent in ramping down from
69 * highest_lpstate 70 * highest_lpstate_idx
70 * @last_sampled_time: Time from boot in ms when global pstates were 71 * @last_sampled_time: Time from boot in ms when global pstates were
71 * last set 72 * last set
72 * @last_lpstate,last_gpstate: Last set values for local and global pstates 73 * @last_lpstate_idx, Last set value of local pstate and global
74 * last_gpstate_idx pstate in terms of cpufreq table index
73 * @timer: Is used for ramping down if cpu goes idle for 75 * @timer: Is used for ramping down if cpu goes idle for
74 * a long time with global pstate held high 76 * a long time with global pstate held high
75 * @gpstate_lock: A spinlock to maintain synchronization between 77 * @gpstate_lock: A spinlock to maintain synchronization between
@@ -77,11 +79,11 @@
77 * governer's target_index calls 79 * governer's target_index calls
78 */ 80 */
79struct global_pstate_info { 81struct global_pstate_info {
80 int highest_lpstate; 82 int highest_lpstate_idx;
81 unsigned int elapsed_time; 83 unsigned int elapsed_time;
82 unsigned int last_sampled_time; 84 unsigned int last_sampled_time;
83 int last_lpstate; 85 int last_lpstate_idx;
84 int last_gpstate; 86 int last_gpstate_idx;
85 spinlock_t gpstate_lock; 87 spinlock_t gpstate_lock;
86 struct timer_list timer; 88 struct timer_list timer;
87}; 89};
@@ -124,29 +126,47 @@ static int nr_chips;
124static DEFINE_PER_CPU(struct chip *, chip_info); 126static DEFINE_PER_CPU(struct chip *, chip_info);
125 127
126/* 128/*
127 * Note: The set of pstates consists of contiguous integers, the 129 * Note:
128 * smallest of which is indicated by powernv_pstate_info.min, the 130 * The set of pstates consists of contiguous integers.
129 * largest of which is indicated by powernv_pstate_info.max. 131 * powernv_pstate_info stores the index of the frequency table for
132 * max, min and nominal frequencies. It also stores number of
133 * available frequencies.
130 * 134 *
131 * The nominal pstate is the highest non-turbo pstate in this 135 * powernv_pstate_info.nominal indicates the index to the highest
132 * platform. This is indicated by powernv_pstate_info.nominal. 136 * non-turbo frequency.
133 */ 137 */
134static struct powernv_pstate_info { 138static struct powernv_pstate_info {
135 int min; 139 unsigned int min;
136 int max; 140 unsigned int max;
137 int nominal; 141 unsigned int nominal;
138 int nr_pstates; 142 unsigned int nr_pstates;
139} powernv_pstate_info; 143} powernv_pstate_info;
140 144
145/* Use following macros for conversions between pstate_id and index */
146static inline int idx_to_pstate(unsigned int i)
147{
148 return powernv_freqs[i].driver_data;
149}
150
151static inline unsigned int pstate_to_idx(int pstate)
152{
153 /*
154 * abs() is deliberately used so that is works with
155 * both monotonically increasing and decreasing
156 * pstate values
157 */
158 return abs(pstate - idx_to_pstate(powernv_pstate_info.max));
159}
160
141static inline void reset_gpstates(struct cpufreq_policy *policy) 161static inline void reset_gpstates(struct cpufreq_policy *policy)
142{ 162{
143 struct global_pstate_info *gpstates = policy->driver_data; 163 struct global_pstate_info *gpstates = policy->driver_data;
144 164
145 gpstates->highest_lpstate = 0; 165 gpstates->highest_lpstate_idx = 0;
146 gpstates->elapsed_time = 0; 166 gpstates->elapsed_time = 0;
147 gpstates->last_sampled_time = 0; 167 gpstates->last_sampled_time = 0;
148 gpstates->last_lpstate = 0; 168 gpstates->last_lpstate_idx = 0;
149 gpstates->last_gpstate = 0; 169 gpstates->last_gpstate_idx = 0;
150} 170}
151 171
152/* 172/*
@@ -156,9 +176,10 @@ static inline void reset_gpstates(struct cpufreq_policy *policy)
156static int init_powernv_pstates(void) 176static int init_powernv_pstates(void)
157{ 177{
158 struct device_node *power_mgt; 178 struct device_node *power_mgt;
159 int i, pstate_min, pstate_max, pstate_nominal, nr_pstates = 0; 179 int i, nr_pstates = 0;
160 const __be32 *pstate_ids, *pstate_freqs; 180 const __be32 *pstate_ids, *pstate_freqs;
161 u32 len_ids, len_freqs; 181 u32 len_ids, len_freqs;
182 u32 pstate_min, pstate_max, pstate_nominal;
162 183
163 power_mgt = of_find_node_by_path("/ibm,opal/power-mgt"); 184 power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
164 if (!power_mgt) { 185 if (!power_mgt) {
@@ -208,6 +229,7 @@ static int init_powernv_pstates(void)
208 return -ENODEV; 229 return -ENODEV;
209 } 230 }
210 231
232 powernv_pstate_info.nr_pstates = nr_pstates;
211 pr_debug("NR PStates %d\n", nr_pstates); 233 pr_debug("NR PStates %d\n", nr_pstates);
212 for (i = 0; i < nr_pstates; i++) { 234 for (i = 0; i < nr_pstates; i++) {
213 u32 id = be32_to_cpu(pstate_ids[i]); 235 u32 id = be32_to_cpu(pstate_ids[i]);
@@ -216,15 +238,17 @@ static int init_powernv_pstates(void)
216 pr_debug("PState id %d freq %d MHz\n", id, freq); 238 pr_debug("PState id %d freq %d MHz\n", id, freq);
217 powernv_freqs[i].frequency = freq * 1000; /* kHz */ 239 powernv_freqs[i].frequency = freq * 1000; /* kHz */
218 powernv_freqs[i].driver_data = id; 240 powernv_freqs[i].driver_data = id;
241
242 if (id == pstate_max)
243 powernv_pstate_info.max = i;
244 else if (id == pstate_nominal)
245 powernv_pstate_info.nominal = i;
246 else if (id == pstate_min)
247 powernv_pstate_info.min = i;
219 } 248 }
249
220 /* End of list marker entry */ 250 /* End of list marker entry */
221 powernv_freqs[i].frequency = CPUFREQ_TABLE_END; 251 powernv_freqs[i].frequency = CPUFREQ_TABLE_END;
222
223 powernv_pstate_info.min = pstate_min;
224 powernv_pstate_info.max = pstate_max;
225 powernv_pstate_info.nominal = pstate_nominal;
226 powernv_pstate_info.nr_pstates = nr_pstates;
227
228 return 0; 252 return 0;
229} 253}
230 254
@@ -233,12 +257,12 @@ static unsigned int pstate_id_to_freq(int pstate_id)
233{ 257{
234 int i; 258 int i;
235 259
236 i = powernv_pstate_info.max - pstate_id; 260 i = pstate_to_idx(pstate_id);
237 if (i >= powernv_pstate_info.nr_pstates || i < 0) { 261 if (i >= powernv_pstate_info.nr_pstates || i < 0) {
238 pr_warn("PState id %d outside of PState table, " 262 pr_warn("PState id %d outside of PState table, "
239 "reporting nominal id %d instead\n", 263 "reporting nominal id %d instead\n",
240 pstate_id, powernv_pstate_info.nominal); 264 pstate_id, idx_to_pstate(powernv_pstate_info.nominal));
241 i = powernv_pstate_info.max - powernv_pstate_info.nominal; 265 i = powernv_pstate_info.nominal;
242 } 266 }
243 267
244 return powernv_freqs[i].frequency; 268 return powernv_freqs[i].frequency;
@@ -252,7 +276,7 @@ static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy,
252 char *buf) 276 char *buf)
253{ 277{
254 return sprintf(buf, "%u\n", 278 return sprintf(buf, "%u\n",
255 pstate_id_to_freq(powernv_pstate_info.nominal)); 279 powernv_freqs[powernv_pstate_info.nominal].frequency);
256} 280}
257 281
258struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq = 282struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq =
@@ -426,7 +450,7 @@ static void set_pstate(void *data)
426 */ 450 */
427static inline unsigned int get_nominal_index(void) 451static inline unsigned int get_nominal_index(void)
428{ 452{
429 return powernv_pstate_info.max - powernv_pstate_info.nominal; 453 return powernv_pstate_info.nominal;
430} 454}
431 455
432static void powernv_cpufreq_throttle_check(void *data) 456static void powernv_cpufreq_throttle_check(void *data)
@@ -435,20 +459,22 @@ static void powernv_cpufreq_throttle_check(void *data)
435 unsigned int cpu = smp_processor_id(); 459 unsigned int cpu = smp_processor_id();
436 unsigned long pmsr; 460 unsigned long pmsr;
437 int pmsr_pmax; 461 int pmsr_pmax;
462 unsigned int pmsr_pmax_idx;
438 463
439 pmsr = get_pmspr(SPRN_PMSR); 464 pmsr = get_pmspr(SPRN_PMSR);
440 chip = this_cpu_read(chip_info); 465 chip = this_cpu_read(chip_info);
441 466
442 /* Check for Pmax Capping */ 467 /* Check for Pmax Capping */
443 pmsr_pmax = (s8)PMSR_MAX(pmsr); 468 pmsr_pmax = (s8)PMSR_MAX(pmsr);
444 if (pmsr_pmax != powernv_pstate_info.max) { 469 pmsr_pmax_idx = pstate_to_idx(pmsr_pmax);
470 if (pmsr_pmax_idx != powernv_pstate_info.max) {
445 if (chip->throttled) 471 if (chip->throttled)
446 goto next; 472 goto next;
447 chip->throttled = true; 473 chip->throttled = true;
448 if (pmsr_pmax < powernv_pstate_info.nominal) { 474 if (pmsr_pmax_idx > powernv_pstate_info.nominal) {
449 pr_warn_once("CPU %d on Chip %u has Pmax reduced below nominal frequency (%d < %d)\n", 475 pr_warn_once("CPU %d on Chip %u has Pmax(%d) reduced below nominal frequency(%d)\n",
450 cpu, chip->id, pmsr_pmax, 476 cpu, chip->id, pmsr_pmax,
451 powernv_pstate_info.nominal); 477 idx_to_pstate(powernv_pstate_info.nominal));
452 chip->throttle_sub_turbo++; 478 chip->throttle_sub_turbo++;
453 } else { 479 } else {
454 chip->throttle_turbo++; 480 chip->throttle_turbo++;
@@ -484,34 +510,35 @@ next:
484 510
485/** 511/**
486 * calc_global_pstate - Calculate global pstate 512 * calc_global_pstate - Calculate global pstate
487 * @elapsed_time: Elapsed time in milliseconds 513 * @elapsed_time: Elapsed time in milliseconds
488 * @local_pstate: New local pstate 514 * @local_pstate_idx: New local pstate
489 * @highest_lpstate: pstate from which its ramping down 515 * @highest_lpstate_idx: pstate from which its ramping down
490 * 516 *
491 * Finds the appropriate global pstate based on the pstate from which its 517 * Finds the appropriate global pstate based on the pstate from which its
492 * ramping down and the time elapsed in ramping down. It follows a quadratic 518 * ramping down and the time elapsed in ramping down. It follows a quadratic
493 * equation which ensures that it reaches ramping down to pmin in 5sec. 519 * equation which ensures that it reaches ramping down to pmin in 5sec.
494 */ 520 */
495static inline int calc_global_pstate(unsigned int elapsed_time, 521static inline int calc_global_pstate(unsigned int elapsed_time,
496 int highest_lpstate, int local_pstate) 522 int highest_lpstate_idx,
523 int local_pstate_idx)
497{ 524{
498 int pstate_diff; 525 int index_diff;
499 526
500 /* 527 /*
501 * Using ramp_down_percent we get the percentage of rampdown 528 * Using ramp_down_percent we get the percentage of rampdown
502 * that we are expecting to be dropping. Difference between 529 * that we are expecting to be dropping. Difference between
503 * highest_lpstate and powernv_pstate_info.min will give a absolute 530 * highest_lpstate_idx and powernv_pstate_info.min will give a absolute
504 * number of how many pstates we will drop eventually by the end of 531 * number of how many pstates we will drop eventually by the end of
505 * 5 seconds, then just scale it get the number pstates to be dropped. 532 * 5 seconds, then just scale it get the number pstates to be dropped.
506 */ 533 */
507 pstate_diff = ((int)ramp_down_percent(elapsed_time) * 534 index_diff = ((int)ramp_down_percent(elapsed_time) *
508 (highest_lpstate - powernv_pstate_info.min)) / 100; 535 (powernv_pstate_info.min - highest_lpstate_idx)) / 100;
509 536
510 /* Ensure that global pstate is >= to local pstate */ 537 /* Ensure that global pstate is >= to local pstate */
511 if (highest_lpstate - pstate_diff < local_pstate) 538 if (highest_lpstate_idx + index_diff >= local_pstate_idx)
512 return local_pstate; 539 return local_pstate_idx;
513 else 540 else
514 return highest_lpstate - pstate_diff; 541 return highest_lpstate_idx + index_diff;
515} 542}
516 543
517static inline void queue_gpstate_timer(struct global_pstate_info *gpstates) 544static inline void queue_gpstate_timer(struct global_pstate_info *gpstates)
@@ -547,7 +574,7 @@ void gpstate_timer_handler(unsigned long data)
547{ 574{
548 struct cpufreq_policy *policy = (struct cpufreq_policy *)data; 575 struct cpufreq_policy *policy = (struct cpufreq_policy *)data;
549 struct global_pstate_info *gpstates = policy->driver_data; 576 struct global_pstate_info *gpstates = policy->driver_data;
550 int gpstate_id; 577 int gpstate_idx;
551 unsigned int time_diff = jiffies_to_msecs(jiffies) 578 unsigned int time_diff = jiffies_to_msecs(jiffies)
552 - gpstates->last_sampled_time; 579 - gpstates->last_sampled_time;
553 struct powernv_smp_call_data freq_data; 580 struct powernv_smp_call_data freq_data;
@@ -557,29 +584,29 @@ void gpstate_timer_handler(unsigned long data)
557 584
558 gpstates->last_sampled_time += time_diff; 585 gpstates->last_sampled_time += time_diff;
559 gpstates->elapsed_time += time_diff; 586 gpstates->elapsed_time += time_diff;
560 freq_data.pstate_id = gpstates->last_lpstate; 587 freq_data.pstate_id = idx_to_pstate(gpstates->last_lpstate_idx);
561 588
562 if ((gpstates->last_gpstate == freq_data.pstate_id) || 589 if ((gpstates->last_gpstate_idx == gpstates->last_lpstate_idx) ||
563 (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME)) { 590 (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME)) {
564 gpstate_id = freq_data.pstate_id; 591 gpstate_idx = pstate_to_idx(freq_data.pstate_id);
565 reset_gpstates(policy); 592 reset_gpstates(policy);
566 gpstates->highest_lpstate = freq_data.pstate_id; 593 gpstates->highest_lpstate_idx = gpstate_idx;
567 } else { 594 } else {
568 gpstate_id = calc_global_pstate(gpstates->elapsed_time, 595 gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
569 gpstates->highest_lpstate, 596 gpstates->highest_lpstate_idx,
570 freq_data.pstate_id); 597 freq_data.pstate_id);
571 } 598 }
572 599
573 /* 600 /*
574 * If local pstate is equal to global pstate, rampdown is over 601 * If local pstate is equal to global pstate, rampdown is over
575 * So timer is not required to be queued. 602 * So timer is not required to be queued.
576 */ 603 */
577 if (gpstate_id != freq_data.pstate_id) 604 if (gpstate_idx != gpstates->last_lpstate_idx)
578 queue_gpstate_timer(gpstates); 605 queue_gpstate_timer(gpstates);
579 606
580 freq_data.gpstate_id = gpstate_id; 607 freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
581 gpstates->last_gpstate = freq_data.gpstate_id; 608 gpstates->last_gpstate_idx = pstate_to_idx(freq_data.gpstate_id);
582 gpstates->last_lpstate = freq_data.pstate_id; 609 gpstates->last_lpstate_idx = pstate_to_idx(freq_data.pstate_id);
583 610
584 spin_unlock(&gpstates->gpstate_lock); 611 spin_unlock(&gpstates->gpstate_lock);
585 612
@@ -596,7 +623,7 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
596 unsigned int new_index) 623 unsigned int new_index)
597{ 624{
598 struct powernv_smp_call_data freq_data; 625 struct powernv_smp_call_data freq_data;
599 unsigned int cur_msec, gpstate_id; 626 unsigned int cur_msec, gpstate_idx;
600 struct global_pstate_info *gpstates = policy->driver_data; 627 struct global_pstate_info *gpstates = policy->driver_data;
601 628
602 if (unlikely(rebooting) && new_index != get_nominal_index()) 629 if (unlikely(rebooting) && new_index != get_nominal_index())
@@ -608,15 +635,15 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
608 cur_msec = jiffies_to_msecs(get_jiffies_64()); 635 cur_msec = jiffies_to_msecs(get_jiffies_64());
609 636
610 spin_lock(&gpstates->gpstate_lock); 637 spin_lock(&gpstates->gpstate_lock);
611 freq_data.pstate_id = powernv_freqs[new_index].driver_data; 638 freq_data.pstate_id = idx_to_pstate(new_index);
612 639
613 if (!gpstates->last_sampled_time) { 640 if (!gpstates->last_sampled_time) {
614 gpstate_id = freq_data.pstate_id; 641 gpstate_idx = new_index;
615 gpstates->highest_lpstate = freq_data.pstate_id; 642 gpstates->highest_lpstate_idx = new_index;
616 goto gpstates_done; 643 goto gpstates_done;
617 } 644 }
618 645
619 if (gpstates->last_gpstate > freq_data.pstate_id) { 646 if (gpstates->last_gpstate_idx < new_index) {
620 gpstates->elapsed_time += cur_msec - 647 gpstates->elapsed_time += cur_msec -
621 gpstates->last_sampled_time; 648 gpstates->last_sampled_time;
622 649
@@ -627,34 +654,34 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
627 */ 654 */
628 if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) { 655 if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
629 reset_gpstates(policy); 656 reset_gpstates(policy);
630 gpstates->highest_lpstate = freq_data.pstate_id; 657 gpstates->highest_lpstate_idx = new_index;
631 gpstate_id = freq_data.pstate_id; 658 gpstate_idx = new_index;
632 } else { 659 } else {
633 /* Elaspsed_time is less than 5 seconds, continue to rampdown */ 660 /* Elaspsed_time is less than 5 seconds, continue to rampdown */
634 gpstate_id = calc_global_pstate(gpstates->elapsed_time, 661 gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
635 gpstates->highest_lpstate, 662 gpstates->highest_lpstate_idx,
636 freq_data.pstate_id); 663 new_index);
637 } 664 }
638 } else { 665 } else {
639 reset_gpstates(policy); 666 reset_gpstates(policy);
640 gpstates->highest_lpstate = freq_data.pstate_id; 667 gpstates->highest_lpstate_idx = new_index;
641 gpstate_id = freq_data.pstate_id; 668 gpstate_idx = new_index;
642 } 669 }
643 670
644 /* 671 /*
645 * If local pstate is equal to global pstate, rampdown is over 672 * If local pstate is equal to global pstate, rampdown is over
646 * So timer is not required to be queued. 673 * So timer is not required to be queued.
647 */ 674 */
648 if (gpstate_id != freq_data.pstate_id) 675 if (gpstate_idx != new_index)
649 queue_gpstate_timer(gpstates); 676 queue_gpstate_timer(gpstates);
650 else 677 else
651 del_timer_sync(&gpstates->timer); 678 del_timer_sync(&gpstates->timer);
652 679
653gpstates_done: 680gpstates_done:
654 freq_data.gpstate_id = gpstate_id; 681 freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
655 gpstates->last_sampled_time = cur_msec; 682 gpstates->last_sampled_time = cur_msec;
656 gpstates->last_gpstate = freq_data.gpstate_id; 683 gpstates->last_gpstate_idx = gpstate_idx;
657 gpstates->last_lpstate = freq_data.pstate_id; 684 gpstates->last_lpstate_idx = new_index;
658 685
659 spin_unlock(&gpstates->gpstate_lock); 686 spin_unlock(&gpstates->gpstate_lock);
660 687
@@ -760,9 +787,7 @@ void powernv_cpufreq_work_fn(struct work_struct *work)
760 struct cpufreq_policy policy; 787 struct cpufreq_policy policy;
761 788
762 cpufreq_get_policy(&policy, cpu); 789 cpufreq_get_policy(&policy, cpu);
763 cpufreq_frequency_table_target(&policy, policy.freq_table, 790 index = cpufreq_table_find_index_c(&policy, policy.cur);
764 policy.cur,
765 CPUFREQ_RELATION_C, &index);
766 powernv_cpufreq_target_index(&policy, index); 791 powernv_cpufreq_target_index(&policy, index);
767 cpumask_andnot(&mask, &mask, policy.cpus); 792 cpumask_andnot(&mask, &mask, policy.cpus);
768 } 793 }
@@ -848,8 +873,8 @@ static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy)
848 struct powernv_smp_call_data freq_data; 873 struct powernv_smp_call_data freq_data;
849 struct global_pstate_info *gpstates = policy->driver_data; 874 struct global_pstate_info *gpstates = policy->driver_data;
850 875
851 freq_data.pstate_id = powernv_pstate_info.min; 876 freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min);
852 freq_data.gpstate_id = powernv_pstate_info.min; 877 freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min);
853 smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1); 878 smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1);
854 del_timer_sync(&gpstates->timer); 879 del_timer_sync(&gpstates->timer);
855} 880}
diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
index 7c4cd5c634f2..dc112481a408 100644
--- a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
+++ b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
@@ -94,7 +94,7 @@ static int pmi_notifier(struct notifier_block *nb,
94 unsigned long event, void *data) 94 unsigned long event, void *data)
95{ 95{
96 struct cpufreq_policy *policy = data; 96 struct cpufreq_policy *policy = data;
97 struct cpufreq_frequency_table *cbe_freqs; 97 struct cpufreq_frequency_table *cbe_freqs = policy->freq_table;
98 u8 node; 98 u8 node;
99 99
100 /* Should this really be called for CPUFREQ_ADJUST and CPUFREQ_NOTIFY 100 /* Should this really be called for CPUFREQ_ADJUST and CPUFREQ_NOTIFY
@@ -103,7 +103,6 @@ static int pmi_notifier(struct notifier_block *nb,
103 if (event == CPUFREQ_START) 103 if (event == CPUFREQ_START)
104 return 0; 104 return 0;
105 105
106 cbe_freqs = cpufreq_frequency_get_table(policy->cpu);
107 node = cbe_cpu_to_node(policy->cpu); 106 node = cbe_cpu_to_node(policy->cpu);
108 107
109 pr_debug("got notified, event=%lu, node=%u\n", event, node); 108 pr_debug("got notified, event=%lu, node=%u\n", event, node);
diff --git a/drivers/cpufreq/s3c24xx-cpufreq.c b/drivers/cpufreq/s3c24xx-cpufreq.c
index ae8eaed77b70..7b596fa38ad2 100644
--- a/drivers/cpufreq/s3c24xx-cpufreq.c
+++ b/drivers/cpufreq/s3c24xx-cpufreq.c
@@ -293,12 +293,8 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy,
293 __func__, policy, target_freq, relation); 293 __func__, policy, target_freq, relation);
294 294
295 if (ftab) { 295 if (ftab) {
296 if (cpufreq_frequency_table_target(policy, ftab, 296 index = cpufreq_frequency_table_target(policy, target_freq,
297 target_freq, relation, 297 relation);
298 &index)) {
299 s3c_freq_dbg("%s: table failed\n", __func__);
300 return -EINVAL;
301 }
302 298
303 s3c_freq_dbg("%s: adjust %d to entry %d (%u)\n", __func__, 299 s3c_freq_dbg("%s: adjust %d to entry %d (%u)\n", __func__,
304 target_freq, index, ftab[index].frequency); 300 target_freq, index, ftab[index].frequency);
@@ -315,7 +311,6 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy,
315 pll = NULL; 311 pll = NULL;
316 } else { 312 } else {
317 struct cpufreq_policy tmp_policy; 313 struct cpufreq_policy tmp_policy;
318 int ret;
319 314
320 /* we keep the cpu pll table in Hz, to ensure we get an 315 /* we keep the cpu pll table in Hz, to ensure we get an
321 * accurate value for the PLL output. */ 316 * accurate value for the PLL output. */
@@ -323,20 +318,14 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy,
323 tmp_policy.min = policy->min * 1000; 318 tmp_policy.min = policy->min * 1000;
324 tmp_policy.max = policy->max * 1000; 319 tmp_policy.max = policy->max * 1000;
325 tmp_policy.cpu = policy->cpu; 320 tmp_policy.cpu = policy->cpu;
321 tmp_policy.freq_table = pll_reg;
326 322
327 /* cpufreq_frequency_table_target uses a pointer to 'index' 323 /* cpufreq_frequency_table_target returns the index
328 * which is the number of the table entry, not the value of 324 * of the table entry, not the value of
329 * the table entry's index field. */ 325 * the table entry's index field. */
330 326
331 ret = cpufreq_frequency_table_target(&tmp_policy, pll_reg, 327 index = cpufreq_frequency_table_target(&tmp_policy, target_freq,
332 target_freq, relation, 328 relation);
333 &index);
334
335 if (ret < 0) {
336 pr_err("%s: no PLL available\n", __func__);
337 goto err_notpossible;
338 }
339
340 pll = pll_reg + index; 329 pll = pll_reg + index;
341 330
342 s3c_freq_dbg("%s: target %u => %u\n", 331 s3c_freq_dbg("%s: target %u => %u\n",
@@ -346,10 +335,6 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy,
346 } 335 }
347 336
348 return s3c_cpufreq_settarget(policy, target_freq, pll); 337 return s3c_cpufreq_settarget(policy, target_freq, pll);
349
350 err_notpossible:
351 pr_err("no compatible settings for %d\n", target_freq);
352 return -EINVAL;
353} 338}
354 339
355struct clk *s3c_cpufreq_clk_get(struct device *dev, const char *name) 340struct clk *s3c_cpufreq_clk_get(struct device *dev, const char *name)
@@ -571,11 +556,7 @@ static int s3c_cpufreq_build_freq(void)
571{ 556{
572 int size, ret; 557 int size, ret;
573 558
574 if (!cpu_cur.info->calc_freqtable)
575 return -EINVAL;
576
577 kfree(ftab); 559 kfree(ftab);
578 ftab = NULL;
579 560
580 size = cpu_cur.info->calc_freqtable(&cpu_cur, NULL, 0); 561 size = cpu_cur.info->calc_freqtable(&cpu_cur, NULL, 0);
581 size++; 562 size++;
diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c
index 06d85917b6d5..9e07588ea9f5 100644
--- a/drivers/cpufreq/s5pv210-cpufreq.c
+++ b/drivers/cpufreq/s5pv210-cpufreq.c
@@ -246,12 +246,7 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index)
246 new_freq = s5pv210_freq_table[index].frequency; 246 new_freq = s5pv210_freq_table[index].frequency;
247 247
248 /* Finding current running level index */ 248 /* Finding current running level index */
249 if (cpufreq_frequency_table_target(policy, s5pv210_freq_table, 249 priv_index = cpufreq_table_find_index_h(policy, old_freq);
250 old_freq, CPUFREQ_RELATION_H,
251 &priv_index)) {
252 ret = -EINVAL;
253 goto exit;
254 }
255 250
256 arm_volt = dvs_conf[index].arm_volt; 251 arm_volt = dvs_conf[index].arm_volt;
257 int_volt = dvs_conf[index].int_volt; 252 int_volt = dvs_conf[index].int_volt;
diff --git a/drivers/thermal/cpu_cooling.c b/drivers/thermal/cpu_cooling.c
index 5b4b47ed948b..3788ed74c9ab 100644
--- a/drivers/thermal/cpu_cooling.c
+++ b/drivers/thermal/cpu_cooling.c
@@ -787,22 +787,34 @@ __cpufreq_cooling_register(struct device_node *np,
787 const struct cpumask *clip_cpus, u32 capacitance, 787 const struct cpumask *clip_cpus, u32 capacitance,
788 get_static_t plat_static_func) 788 get_static_t plat_static_func)
789{ 789{
790 struct cpufreq_policy *policy;
790 struct thermal_cooling_device *cool_dev; 791 struct thermal_cooling_device *cool_dev;
791 struct cpufreq_cooling_device *cpufreq_dev; 792 struct cpufreq_cooling_device *cpufreq_dev;
792 char dev_name[THERMAL_NAME_LENGTH]; 793 char dev_name[THERMAL_NAME_LENGTH];
793 struct cpufreq_frequency_table *pos, *table; 794 struct cpufreq_frequency_table *pos, *table;
795 struct cpumask temp_mask;
794 unsigned int freq, i, num_cpus; 796 unsigned int freq, i, num_cpus;
795 int ret; 797 int ret;
796 798
797 table = cpufreq_frequency_get_table(cpumask_first(clip_cpus)); 799 cpumask_and(&temp_mask, clip_cpus, cpu_online_mask);
800 policy = cpufreq_cpu_get(cpumask_first(&temp_mask));
801 if (!policy) {
802 pr_debug("%s: CPUFreq policy not found\n", __func__);
803 return ERR_PTR(-EPROBE_DEFER);
804 }
805
806 table = policy->freq_table;
798 if (!table) { 807 if (!table) {
799 pr_debug("%s: CPUFreq table not found\n", __func__); 808 pr_debug("%s: CPUFreq table not found\n", __func__);
800 return ERR_PTR(-EPROBE_DEFER); 809 cool_dev = ERR_PTR(-ENODEV);
810 goto put_policy;
801 } 811 }
802 812
803 cpufreq_dev = kzalloc(sizeof(*cpufreq_dev), GFP_KERNEL); 813 cpufreq_dev = kzalloc(sizeof(*cpufreq_dev), GFP_KERNEL);
804 if (!cpufreq_dev) 814 if (!cpufreq_dev) {
805 return ERR_PTR(-ENOMEM); 815 cool_dev = ERR_PTR(-ENOMEM);
816 goto put_policy;
817 }
806 818
807 num_cpus = cpumask_weight(clip_cpus); 819 num_cpus = cpumask_weight(clip_cpus);
808 cpufreq_dev->time_in_idle = kcalloc(num_cpus, 820 cpufreq_dev->time_in_idle = kcalloc(num_cpus,
@@ -892,7 +904,7 @@ __cpufreq_cooling_register(struct device_node *np,
892 CPUFREQ_POLICY_NOTIFIER); 904 CPUFREQ_POLICY_NOTIFIER);
893 mutex_unlock(&cooling_cpufreq_lock); 905 mutex_unlock(&cooling_cpufreq_lock);
894 906
895 return cool_dev; 907 goto put_policy;
896 908
897remove_idr: 909remove_idr:
898 release_idr(&cpufreq_idr, cpufreq_dev->id); 910 release_idr(&cpufreq_idr, cpufreq_dev->id);
@@ -906,6 +918,8 @@ free_time_in_idle:
906 kfree(cpufreq_dev->time_in_idle); 918 kfree(cpufreq_dev->time_in_idle);
907free_cdev: 919free_cdev:
908 kfree(cpufreq_dev); 920 kfree(cpufreq_dev);
921put_policy:
922 cpufreq_cpu_put(policy);
909 923
910 return cool_dev; 924 return cool_dev;
911} 925}
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index 4e81e08db752..631ba33bbe9f 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -36,6 +36,12 @@
36 36
37struct cpufreq_governor; 37struct cpufreq_governor;
38 38
39enum cpufreq_table_sorting {
40 CPUFREQ_TABLE_UNSORTED,
41 CPUFREQ_TABLE_SORTED_ASCENDING,
42 CPUFREQ_TABLE_SORTED_DESCENDING
43};
44
39struct cpufreq_freqs { 45struct cpufreq_freqs {
40 unsigned int cpu; /* cpu nr */ 46 unsigned int cpu; /* cpu nr */
41 unsigned int old; 47 unsigned int old;
@@ -87,6 +93,7 @@ struct cpufreq_policy {
87 93
88 struct cpufreq_user_policy user_policy; 94 struct cpufreq_user_policy user_policy;
89 struct cpufreq_frequency_table *freq_table; 95 struct cpufreq_frequency_table *freq_table;
96 enum cpufreq_table_sorting freq_table_sorted;
90 97
91 struct list_head policy_list; 98 struct list_head policy_list;
92 struct kobject kobj; 99 struct kobject kobj;
@@ -113,6 +120,10 @@ struct cpufreq_policy {
113 bool fast_switch_possible; 120 bool fast_switch_possible;
114 bool fast_switch_enabled; 121 bool fast_switch_enabled;
115 122
123 /* Cached frequency lookup from cpufreq_driver_resolve_freq. */
124 unsigned int cached_target_freq;
125 int cached_resolved_idx;
126
116 /* Synchronization for frequency transitions */ 127 /* Synchronization for frequency transitions */
117 bool transition_ongoing; /* Tracks transition status */ 128 bool transition_ongoing; /* Tracks transition status */
118 spinlock_t transition_lock; 129 spinlock_t transition_lock;
@@ -185,6 +196,18 @@ static inline unsigned int cpufreq_quick_get_max(unsigned int cpu)
185static inline void disable_cpufreq(void) { } 196static inline void disable_cpufreq(void) { }
186#endif 197#endif
187 198
199#ifdef CONFIG_CPU_FREQ_STAT
200void cpufreq_stats_create_table(struct cpufreq_policy *policy);
201void cpufreq_stats_free_table(struct cpufreq_policy *policy);
202void cpufreq_stats_record_transition(struct cpufreq_policy *policy,
203 unsigned int new_freq);
204#else
205static inline void cpufreq_stats_create_table(struct cpufreq_policy *policy) { }
206static inline void cpufreq_stats_free_table(struct cpufreq_policy *policy) { }
207static inline void cpufreq_stats_record_transition(struct cpufreq_policy *policy,
208 unsigned int new_freq) { }
209#endif /* CONFIG_CPU_FREQ_STAT */
210
188/********************************************************************* 211/*********************************************************************
189 * CPUFREQ DRIVER INTERFACE * 212 * CPUFREQ DRIVER INTERFACE *
190 *********************************************************************/ 213 *********************************************************************/
@@ -251,6 +274,16 @@ struct cpufreq_driver {
251 unsigned int index); 274 unsigned int index);
252 unsigned int (*fast_switch)(struct cpufreq_policy *policy, 275 unsigned int (*fast_switch)(struct cpufreq_policy *policy,
253 unsigned int target_freq); 276 unsigned int target_freq);
277
278 /*
279 * Caches and returns the lowest driver-supported frequency greater than
280 * or equal to the target frequency, subject to any driver limitations.
281 * Does not set the frequency. Only to be implemented for drivers with
282 * target().
283 */
284 unsigned int (*resolve_freq)(struct cpufreq_policy *policy,
285 unsigned int target_freq);
286
254 /* 287 /*
255 * Only for drivers with target_index() and CPUFREQ_ASYNC_NOTIFICATION 288 * Only for drivers with target_index() and CPUFREQ_ASYNC_NOTIFICATION
256 * unset. 289 * unset.
@@ -455,18 +488,13 @@ static inline unsigned long cpufreq_scale(unsigned long old, u_int div,
455#define MIN_LATENCY_MULTIPLIER (20) 488#define MIN_LATENCY_MULTIPLIER (20)
456#define TRANSITION_LATENCY_LIMIT (10 * 1000 * 1000) 489#define TRANSITION_LATENCY_LIMIT (10 * 1000 * 1000)
457 490
458/* Governor Events */
459#define CPUFREQ_GOV_START 1
460#define CPUFREQ_GOV_STOP 2
461#define CPUFREQ_GOV_LIMITS 3
462#define CPUFREQ_GOV_POLICY_INIT 4
463#define CPUFREQ_GOV_POLICY_EXIT 5
464
465struct cpufreq_governor { 491struct cpufreq_governor {
466 char name[CPUFREQ_NAME_LEN]; 492 char name[CPUFREQ_NAME_LEN];
467 int initialized; 493 int (*init)(struct cpufreq_policy *policy);
468 int (*governor) (struct cpufreq_policy *policy, 494 void (*exit)(struct cpufreq_policy *policy);
469 unsigned int event); 495 int (*start)(struct cpufreq_policy *policy);
496 void (*stop)(struct cpufreq_policy *policy);
497 void (*limits)(struct cpufreq_policy *policy);
470 ssize_t (*show_setspeed) (struct cpufreq_policy *policy, 498 ssize_t (*show_setspeed) (struct cpufreq_policy *policy,
471 char *buf); 499 char *buf);
472 int (*store_setspeed) (struct cpufreq_policy *policy, 500 int (*store_setspeed) (struct cpufreq_policy *policy,
@@ -487,12 +515,22 @@ int cpufreq_driver_target(struct cpufreq_policy *policy,
487int __cpufreq_driver_target(struct cpufreq_policy *policy, 515int __cpufreq_driver_target(struct cpufreq_policy *policy,
488 unsigned int target_freq, 516 unsigned int target_freq,
489 unsigned int relation); 517 unsigned int relation);
518unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
519 unsigned int target_freq);
490int cpufreq_register_governor(struct cpufreq_governor *governor); 520int cpufreq_register_governor(struct cpufreq_governor *governor);
491void cpufreq_unregister_governor(struct cpufreq_governor *governor); 521void cpufreq_unregister_governor(struct cpufreq_governor *governor);
492 522
493struct cpufreq_governor *cpufreq_default_governor(void); 523struct cpufreq_governor *cpufreq_default_governor(void);
494struct cpufreq_governor *cpufreq_fallback_governor(void); 524struct cpufreq_governor *cpufreq_fallback_governor(void);
495 525
526static inline void cpufreq_policy_apply_limits(struct cpufreq_policy *policy)
527{
528 if (policy->max < policy->cur)
529 __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H);
530 else if (policy->min > policy->cur)
531 __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L);
532}
533
496/* Governor attribute set */ 534/* Governor attribute set */
497struct gov_attr_set { 535struct gov_attr_set {
498 struct kobject kobj; 536 struct kobject kobj;
@@ -582,11 +620,9 @@ int cpufreq_frequency_table_verify(struct cpufreq_policy *policy,
582 struct cpufreq_frequency_table *table); 620 struct cpufreq_frequency_table *table);
583int cpufreq_generic_frequency_table_verify(struct cpufreq_policy *policy); 621int cpufreq_generic_frequency_table_verify(struct cpufreq_policy *policy);
584 622
585int cpufreq_frequency_table_target(struct cpufreq_policy *policy, 623int cpufreq_table_index_unsorted(struct cpufreq_policy *policy,
586 struct cpufreq_frequency_table *table, 624 unsigned int target_freq,
587 unsigned int target_freq, 625 unsigned int relation);
588 unsigned int relation,
589 unsigned int *index);
590int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy, 626int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy,
591 unsigned int freq); 627 unsigned int freq);
592 628
@@ -597,6 +633,227 @@ int cpufreq_boost_trigger_state(int state);
597int cpufreq_boost_enabled(void); 633int cpufreq_boost_enabled(void);
598int cpufreq_enable_boost_support(void); 634int cpufreq_enable_boost_support(void);
599bool policy_has_boost_freq(struct cpufreq_policy *policy); 635bool policy_has_boost_freq(struct cpufreq_policy *policy);
636
637/* Find lowest freq at or above target in a table in ascending order */
638static inline int cpufreq_table_find_index_al(struct cpufreq_policy *policy,
639 unsigned int target_freq)
640{
641 struct cpufreq_frequency_table *table = policy->freq_table;
642 unsigned int freq;
643 int i, best = -1;
644
645 for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) {
646 freq = table[i].frequency;
647
648 if (freq >= target_freq)
649 return i;
650
651 best = i;
652 }
653
654 return best;
655}
656
657/* Find lowest freq at or above target in a table in descending order */
658static inline int cpufreq_table_find_index_dl(struct cpufreq_policy *policy,
659 unsigned int target_freq)
660{
661 struct cpufreq_frequency_table *table = policy->freq_table;
662 unsigned int freq;
663 int i, best = -1;
664
665 for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) {
666 freq = table[i].frequency;
667
668 if (freq == target_freq)
669 return i;
670
671 if (freq > target_freq) {
672 best = i;
673 continue;
674 }
675
676 /* No freq found above target_freq */
677 if (best == -1)
678 return i;
679
680 return best;
681 }
682
683 return best;
684}
685
686/* Works only on sorted freq-tables */
687static inline int cpufreq_table_find_index_l(struct cpufreq_policy *policy,
688 unsigned int target_freq)
689{
690 target_freq = clamp_val(target_freq, policy->min, policy->max);
691
692 if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING)
693 return cpufreq_table_find_index_al(policy, target_freq);
694 else
695 return cpufreq_table_find_index_dl(policy, target_freq);
696}
697
698/* Find highest freq at or below target in a table in ascending order */
699static inline int cpufreq_table_find_index_ah(struct cpufreq_policy *policy,
700 unsigned int target_freq)
701{
702 struct cpufreq_frequency_table *table = policy->freq_table;
703 unsigned int freq;
704 int i, best = -1;
705
706 for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) {
707 freq = table[i].frequency;
708
709 if (freq == target_freq)
710 return i;
711
712 if (freq < target_freq) {
713 best = i;
714 continue;
715 }
716
717 /* No freq found below target_freq */
718 if (best == -1)
719 return i;
720
721 return best;
722 }
723
724 return best;
725}
726
727/* Find highest freq at or below target in a table in descending order */
728static inline int cpufreq_table_find_index_dh(struct cpufreq_policy *policy,
729 unsigned int target_freq)
730{
731 struct cpufreq_frequency_table *table = policy->freq_table;
732 unsigned int freq;
733 int i, best = -1;
734
735 for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) {
736 freq = table[i].frequency;
737
738 if (freq <= target_freq)
739 return i;
740
741 best = i;
742 }
743
744 return best;
745}
746
747/* Works only on sorted freq-tables */
748static inline int cpufreq_table_find_index_h(struct cpufreq_policy *policy,
749 unsigned int target_freq)
750{
751 target_freq = clamp_val(target_freq, policy->min, policy->max);
752
753 if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING)
754 return cpufreq_table_find_index_ah(policy, target_freq);
755 else
756 return cpufreq_table_find_index_dh(policy, target_freq);
757}
758
759/* Find closest freq to target in a table in ascending order */
760static inline int cpufreq_table_find_index_ac(struct cpufreq_policy *policy,
761 unsigned int target_freq)
762{
763 struct cpufreq_frequency_table *table = policy->freq_table;
764 unsigned int freq;
765 int i, best = -1;
766
767 for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) {
768 freq = table[i].frequency;
769
770 if (freq == target_freq)
771 return i;
772
773 if (freq < target_freq) {
774 best = i;
775 continue;
776 }
777
778 /* No freq found below target_freq */
779 if (best == -1)
780 return i;
781
782 /* Choose the closest freq */
783 if (target_freq - table[best].frequency > freq - target_freq)
784 return i;
785
786 return best;
787 }
788
789 return best;
790}
791
792/* Find closest freq to target in a table in descending order */
793static inline int cpufreq_table_find_index_dc(struct cpufreq_policy *policy,
794 unsigned int target_freq)
795{
796 struct cpufreq_frequency_table *table = policy->freq_table;
797 unsigned int freq;
798 int i, best = -1;
799
800 for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) {
801 freq = table[i].frequency;
802
803 if (freq == target_freq)
804 return i;
805
806 if (freq > target_freq) {
807 best = i;
808 continue;
809 }
810
811 /* No freq found above target_freq */
812 if (best == -1)
813 return i;
814
815 /* Choose the closest freq */
816 if (table[best].frequency - target_freq > target_freq - freq)
817 return i;
818
819 return best;
820 }
821
822 return best;
823}
824
825/* Works only on sorted freq-tables */
826static inline int cpufreq_table_find_index_c(struct cpufreq_policy *policy,
827 unsigned int target_freq)
828{
829 target_freq = clamp_val(target_freq, policy->min, policy->max);
830
831 if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING)
832 return cpufreq_table_find_index_ac(policy, target_freq);
833 else
834 return cpufreq_table_find_index_dc(policy, target_freq);
835}
836
837static inline int cpufreq_frequency_table_target(struct cpufreq_policy *policy,
838 unsigned int target_freq,
839 unsigned int relation)
840{
841 if (unlikely(policy->freq_table_sorted == CPUFREQ_TABLE_UNSORTED))
842 return cpufreq_table_index_unsorted(policy, target_freq,
843 relation);
844
845 switch (relation) {
846 case CPUFREQ_RELATION_L:
847 return cpufreq_table_find_index_l(policy, target_freq);
848 case CPUFREQ_RELATION_H:
849 return cpufreq_table_find_index_h(policy, target_freq);
850 case CPUFREQ_RELATION_C:
851 return cpufreq_table_find_index_c(policy, target_freq);
852 default:
853 pr_err("%s: Invalid relation: %d\n", __func__, relation);
854 return -EINVAL;
855 }
856}
600#else 857#else
601static inline int cpufreq_boost_trigger_state(int state) 858static inline int cpufreq_boost_trigger_state(int state)
602{ 859{
@@ -617,8 +874,6 @@ static inline bool policy_has_boost_freq(struct cpufreq_policy *policy)
617 return false; 874 return false;
618} 875}
619#endif 876#endif
620/* the following funtion is for cpufreq core use only */
621struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu);
622 877
623/* the following are really really optional */ 878/* the following are really really optional */
624extern struct freq_attr cpufreq_freq_attr_scaling_available_freqs; 879extern struct freq_attr cpufreq_freq_attr_scaling_available_freqs;
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index 14c4aa25cc45..a84641b222c1 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -47,6 +47,8 @@ struct sugov_cpu {
47 struct update_util_data update_util; 47 struct update_util_data update_util;
48 struct sugov_policy *sg_policy; 48 struct sugov_policy *sg_policy;
49 49
50 unsigned int cached_raw_freq;
51
50 /* The fields below are only needed when sharing a policy. */ 52 /* The fields below are only needed when sharing a policy. */
51 unsigned long util; 53 unsigned long util;
52 unsigned long max; 54 unsigned long max;
@@ -106,7 +108,7 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
106 108
107/** 109/**
108 * get_next_freq - Compute a new frequency for a given cpufreq policy. 110 * get_next_freq - Compute a new frequency for a given cpufreq policy.
109 * @policy: cpufreq policy object to compute the new frequency for. 111 * @sg_cpu: schedutil cpu object to compute the new frequency for.
110 * @util: Current CPU utilization. 112 * @util: Current CPU utilization.
111 * @max: CPU capacity. 113 * @max: CPU capacity.
112 * 114 *
@@ -121,14 +123,25 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
121 * next_freq = C * curr_freq * util_raw / max 123 * next_freq = C * curr_freq * util_raw / max
122 * 124 *
123 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8. 125 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
126 *
127 * The lowest driver-supported frequency which is equal or greater than the raw
128 * next_freq (as calculated above) is returned, subject to policy min/max and
129 * cpufreq driver limitations.
124 */ 130 */
125static unsigned int get_next_freq(struct cpufreq_policy *policy, 131static unsigned int get_next_freq(struct sugov_cpu *sg_cpu, unsigned long util,
126 unsigned long util, unsigned long max) 132 unsigned long max)
127{ 133{
134 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
135 struct cpufreq_policy *policy = sg_policy->policy;
128 unsigned int freq = arch_scale_freq_invariant() ? 136 unsigned int freq = arch_scale_freq_invariant() ?
129 policy->cpuinfo.max_freq : policy->cur; 137 policy->cpuinfo.max_freq : policy->cur;
130 138
131 return (freq + (freq >> 2)) * util / max; 139 freq = (freq + (freq >> 2)) * util / max;
140
141 if (freq == sg_cpu->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
142 return sg_policy->next_freq;
143 sg_cpu->cached_raw_freq = freq;
144 return cpufreq_driver_resolve_freq(policy, freq);
132} 145}
133 146
134static void sugov_update_single(struct update_util_data *hook, u64 time, 147static void sugov_update_single(struct update_util_data *hook, u64 time,
@@ -143,13 +156,14 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
143 return; 156 return;
144 157
145 next_f = util == ULONG_MAX ? policy->cpuinfo.max_freq : 158 next_f = util == ULONG_MAX ? policy->cpuinfo.max_freq :
146 get_next_freq(policy, util, max); 159 get_next_freq(sg_cpu, util, max);
147 sugov_update_commit(sg_policy, time, next_f); 160 sugov_update_commit(sg_policy, time, next_f);
148} 161}
149 162
150static unsigned int sugov_next_freq_shared(struct sugov_policy *sg_policy, 163static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
151 unsigned long util, unsigned long max) 164 unsigned long util, unsigned long max)
152{ 165{
166 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
153 struct cpufreq_policy *policy = sg_policy->policy; 167 struct cpufreq_policy *policy = sg_policy->policy;
154 unsigned int max_f = policy->cpuinfo.max_freq; 168 unsigned int max_f = policy->cpuinfo.max_freq;
155 u64 last_freq_update_time = sg_policy->last_freq_update_time; 169 u64 last_freq_update_time = sg_policy->last_freq_update_time;
@@ -189,7 +203,7 @@ static unsigned int sugov_next_freq_shared(struct sugov_policy *sg_policy,
189 } 203 }
190 } 204 }
191 205
192 return get_next_freq(policy, util, max); 206 return get_next_freq(sg_cpu, util, max);
193} 207}
194 208
195static void sugov_update_shared(struct update_util_data *hook, u64 time, 209static void sugov_update_shared(struct update_util_data *hook, u64 time,
@@ -206,7 +220,7 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
206 sg_cpu->last_update = time; 220 sg_cpu->last_update = time;
207 221
208 if (sugov_should_update_freq(sg_policy, time)) { 222 if (sugov_should_update_freq(sg_policy, time)) {
209 next_f = sugov_next_freq_shared(sg_policy, util, max); 223 next_f = sugov_next_freq_shared(sg_cpu, util, max);
210 sugov_update_commit(sg_policy, time, next_f); 224 sugov_update_commit(sg_policy, time, next_f);
211 } 225 }
212 226
@@ -394,7 +408,7 @@ static int sugov_init(struct cpufreq_policy *policy)
394 return ret; 408 return ret;
395} 409}
396 410
397static int sugov_exit(struct cpufreq_policy *policy) 411static void sugov_exit(struct cpufreq_policy *policy)
398{ 412{
399 struct sugov_policy *sg_policy = policy->governor_data; 413 struct sugov_policy *sg_policy = policy->governor_data;
400 struct sugov_tunables *tunables = sg_policy->tunables; 414 struct sugov_tunables *tunables = sg_policy->tunables;
@@ -412,7 +426,6 @@ static int sugov_exit(struct cpufreq_policy *policy)
412 mutex_unlock(&global_tunables_lock); 426 mutex_unlock(&global_tunables_lock);
413 427
414 sugov_policy_free(sg_policy); 428 sugov_policy_free(sg_policy);
415 return 0;
416} 429}
417 430
418static int sugov_start(struct cpufreq_policy *policy) 431static int sugov_start(struct cpufreq_policy *policy)
@@ -434,6 +447,7 @@ static int sugov_start(struct cpufreq_policy *policy)
434 sg_cpu->util = ULONG_MAX; 447 sg_cpu->util = ULONG_MAX;
435 sg_cpu->max = 0; 448 sg_cpu->max = 0;
436 sg_cpu->last_update = 0; 449 sg_cpu->last_update = 0;
450 sg_cpu->cached_raw_freq = 0;
437 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, 451 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
438 sugov_update_shared); 452 sugov_update_shared);
439 } else { 453 } else {
@@ -444,7 +458,7 @@ static int sugov_start(struct cpufreq_policy *policy)
444 return 0; 458 return 0;
445} 459}
446 460
447static int sugov_stop(struct cpufreq_policy *policy) 461static void sugov_stop(struct cpufreq_policy *policy)
448{ 462{
449 struct sugov_policy *sg_policy = policy->governor_data; 463 struct sugov_policy *sg_policy = policy->governor_data;
450 unsigned int cpu; 464 unsigned int cpu;
@@ -456,53 +470,29 @@ static int sugov_stop(struct cpufreq_policy *policy)
456 470
457 irq_work_sync(&sg_policy->irq_work); 471 irq_work_sync(&sg_policy->irq_work);
458 cancel_work_sync(&sg_policy->work); 472 cancel_work_sync(&sg_policy->work);
459 return 0;
460} 473}
461 474
462static int sugov_limits(struct cpufreq_policy *policy) 475static void sugov_limits(struct cpufreq_policy *policy)
463{ 476{
464 struct sugov_policy *sg_policy = policy->governor_data; 477 struct sugov_policy *sg_policy = policy->governor_data;
465 478
466 if (!policy->fast_switch_enabled) { 479 if (!policy->fast_switch_enabled) {
467 mutex_lock(&sg_policy->work_lock); 480 mutex_lock(&sg_policy->work_lock);
468 481 cpufreq_policy_apply_limits(policy);
469 if (policy->max < policy->cur)
470 __cpufreq_driver_target(policy, policy->max,
471 CPUFREQ_RELATION_H);
472 else if (policy->min > policy->cur)
473 __cpufreq_driver_target(policy, policy->min,
474 CPUFREQ_RELATION_L);
475
476 mutex_unlock(&sg_policy->work_lock); 482 mutex_unlock(&sg_policy->work_lock);
477 } 483 }
478 484
479 sg_policy->need_freq_update = true; 485 sg_policy->need_freq_update = true;
480 return 0;
481}
482
483int sugov_governor(struct cpufreq_policy *policy, unsigned int event)
484{
485 if (event == CPUFREQ_GOV_POLICY_INIT) {
486 return sugov_init(policy);
487 } else if (policy->governor_data) {
488 switch (event) {
489 case CPUFREQ_GOV_POLICY_EXIT:
490 return sugov_exit(policy);
491 case CPUFREQ_GOV_START:
492 return sugov_start(policy);
493 case CPUFREQ_GOV_STOP:
494 return sugov_stop(policy);
495 case CPUFREQ_GOV_LIMITS:
496 return sugov_limits(policy);
497 }
498 }
499 return -EINVAL;
500} 486}
501 487
502static struct cpufreq_governor schedutil_gov = { 488static struct cpufreq_governor schedutil_gov = {
503 .name = "schedutil", 489 .name = "schedutil",
504 .governor = sugov_governor,
505 .owner = THIS_MODULE, 490 .owner = THIS_MODULE,
491 .init = sugov_init,
492 .exit = sugov_exit,
493 .start = sugov_start,
494 .stop = sugov_stop,
495 .limits = sugov_limits,
506}; 496};
507 497
508static int __init sugov_module_init(void) 498static int __init sugov_module_init(void)
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-26 02:56:40 -0400