aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/cpufreq
diff options
context:
space:
mode:
authorViresh Kumar <viresh.kumar@linaro.org>2013-03-27 11:58:58 -0400
committerRafael J. Wysocki <rafael.j.wysocki@intel.com>2013-03-31 19:11:34 -0400
commit4d5dcc4211f9def4281eafb54b8ed483862e8135 (patch)
tree7f3c725675ce3042d2d2eb86b0b40f93cef73de3 /drivers/cpufreq
parent7bd353a995d9049262661d85811d6109140582a3 (diff)
cpufreq: governor: Implement per policy instances of governors
Currently, there can't be multiple instances of single governor_type. If we have a multi-package system, where we have multiple instances of struct policy (per package), we can't have multiple instances of same governor. i.e. We can't have multiple instances of ondemand governor for multiple packages. Governors directory in sysfs is created at /sys/devices/system/cpu/cpufreq/ governor-name/. Which again reflects that there can be only one instance of a governor_type in the system. This is a bottleneck for multicluster system, where we want different packages to use same governor type, but with different tunables. This patch uses the infrastructure provided by earlier patch and implements init/exit routines for ondemand and conservative governors. Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org> Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Diffstat (limited to 'drivers/cpufreq')
-rw-r--r--drivers/cpufreq/cpufreq.c15
-rw-r--r--drivers/cpufreq/cpufreq_conservative.c193
-rw-r--r--drivers/cpufreq/cpufreq_governor.c212
-rw-r--r--drivers/cpufreq/cpufreq_governor.h117
-rw-r--r--drivers/cpufreq/cpufreq_ondemand.c263
5 files changed, 530 insertions, 270 deletions
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 08df7a196116..85963fc48a5f 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -128,6 +128,11 @@ void disable_cpufreq(void)
128static LIST_HEAD(cpufreq_governor_list); 128static LIST_HEAD(cpufreq_governor_list);
129static DEFINE_MUTEX(cpufreq_governor_mutex); 129static DEFINE_MUTEX(cpufreq_governor_mutex);
130 130
131bool have_governor_per_policy(void)
132{
133 return cpufreq_driver->have_governor_per_policy;
134}
135
131static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs) 136static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
132{ 137{
133 struct cpufreq_policy *data; 138 struct cpufreq_policy *data;
@@ -1546,10 +1551,12 @@ static int __cpufreq_governor(struct cpufreq_policy *policy,
1546 policy->cpu, event); 1551 policy->cpu, event);
1547 ret = policy->governor->governor(policy, event); 1552 ret = policy->governor->governor(policy, event);
1548 1553
1549 if (event == CPUFREQ_GOV_START) 1554 if (!ret) {
1550 policy->governor->initialized++; 1555 if (event == CPUFREQ_GOV_POLICY_INIT)
1551 else if (event == CPUFREQ_GOV_STOP) 1556 policy->governor->initialized++;
1552 policy->governor->initialized--; 1557 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1558 policy->governor->initialized--;
1559 }
1553 1560
1554 /* we keep one module reference alive for 1561 /* we keep one module reference alive for
1555 each CPU governed by this CPU */ 1562 each CPU governed by this CPU */
diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c
index 4fd0006b1291..98b49462f4e9 100644
--- a/drivers/cpufreq/cpufreq_conservative.c
+++ b/drivers/cpufreq/cpufreq_conservative.c
@@ -20,6 +20,7 @@
20#include <linux/mutex.h> 20#include <linux/mutex.h>
21#include <linux/notifier.h> 21#include <linux/notifier.h>
22#include <linux/percpu-defs.h> 22#include <linux/percpu-defs.h>
23#include <linux/slab.h>
23#include <linux/sysfs.h> 24#include <linux/sysfs.h>
24#include <linux/types.h> 25#include <linux/types.h>
25 26
@@ -31,17 +32,8 @@
31#define DEF_SAMPLING_DOWN_FACTOR (1) 32#define DEF_SAMPLING_DOWN_FACTOR (1)
32#define MAX_SAMPLING_DOWN_FACTOR (10) 33#define MAX_SAMPLING_DOWN_FACTOR (10)
33 34
34static struct dbs_data cs_dbs_data;
35static DEFINE_PER_CPU(struct cs_cpu_dbs_info_s, cs_cpu_dbs_info); 35static DEFINE_PER_CPU(struct cs_cpu_dbs_info_s, cs_cpu_dbs_info);
36 36
37static struct cs_dbs_tuners cs_tuners = {
38 .up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
39 .down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD,
40 .sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR,
41 .ignore_nice = 0,
42 .freq_step = 5,
43};
44
45/* 37/*
46 * Every sampling_rate, we check, if current idle time is less than 20% 38 * Every sampling_rate, we check, if current idle time is less than 20%
47 * (default), then we try to increase frequency Every sampling_rate * 39 * (default), then we try to increase frequency Every sampling_rate *
@@ -55,24 +47,26 @@ static void cs_check_cpu(int cpu, unsigned int load)
55{ 47{
56 struct cs_cpu_dbs_info_s *dbs_info = &per_cpu(cs_cpu_dbs_info, cpu); 48 struct cs_cpu_dbs_info_s *dbs_info = &per_cpu(cs_cpu_dbs_info, cpu);
57 struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; 49 struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy;
50 struct dbs_data *dbs_data = policy->governor_data;
51 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
58 unsigned int freq_target; 52 unsigned int freq_target;
59 53
60 /* 54 /*
61 * break out if we 'cannot' reduce the speed as the user might 55 * break out if we 'cannot' reduce the speed as the user might
62 * want freq_step to be zero 56 * want freq_step to be zero
63 */ 57 */
64 if (cs_tuners.freq_step == 0) 58 if (cs_tuners->freq_step == 0)
65 return; 59 return;
66 60
67 /* Check for frequency increase */ 61 /* Check for frequency increase */
68 if (load > cs_tuners.up_threshold) { 62 if (load > cs_tuners->up_threshold) {
69 dbs_info->down_skip = 0; 63 dbs_info->down_skip = 0;
70 64
71 /* if we are already at full speed then break out early */ 65 /* if we are already at full speed then break out early */
72 if (dbs_info->requested_freq == policy->max) 66 if (dbs_info->requested_freq == policy->max)
73 return; 67 return;
74 68
75 freq_target = (cs_tuners.freq_step * policy->max) / 100; 69 freq_target = (cs_tuners->freq_step * policy->max) / 100;
76 70
77 /* max freq cannot be less than 100. But who knows.... */ 71 /* max freq cannot be less than 100. But who knows.... */
78 if (unlikely(freq_target == 0)) 72 if (unlikely(freq_target == 0))
@@ -92,8 +86,8 @@ static void cs_check_cpu(int cpu, unsigned int load)
92 * support the current CPU usage without triggering the up policy. To be 86 * support the current CPU usage without triggering the up policy. To be
93 * safe, we focus 10 points under the threshold. 87 * safe, we focus 10 points under the threshold.
94 */ 88 */
95 if (load < (cs_tuners.down_threshold - 10)) { 89 if (load < (cs_tuners->down_threshold - 10)) {
96 freq_target = (cs_tuners.freq_step * policy->max) / 100; 90 freq_target = (cs_tuners->freq_step * policy->max) / 100;
97 91
98 dbs_info->requested_freq -= freq_target; 92 dbs_info->requested_freq -= freq_target;
99 if (dbs_info->requested_freq < policy->min) 93 if (dbs_info->requested_freq < policy->min)
@@ -119,11 +113,13 @@ static void cs_dbs_timer(struct work_struct *work)
119 unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; 113 unsigned int cpu = dbs_info->cdbs.cur_policy->cpu;
120 struct cs_cpu_dbs_info_s *core_dbs_info = &per_cpu(cs_cpu_dbs_info, 114 struct cs_cpu_dbs_info_s *core_dbs_info = &per_cpu(cs_cpu_dbs_info,
121 cpu); 115 cpu);
122 int delay = delay_for_sampling_rate(cs_tuners.sampling_rate); 116 struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data;
117 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
118 int delay = delay_for_sampling_rate(cs_tuners->sampling_rate);
123 119
124 mutex_lock(&core_dbs_info->cdbs.timer_mutex); 120 mutex_lock(&core_dbs_info->cdbs.timer_mutex);
125 if (need_load_eval(&core_dbs_info->cdbs, cs_tuners.sampling_rate)) 121 if (need_load_eval(&core_dbs_info->cdbs, cs_tuners->sampling_rate))
126 dbs_check_cpu(&cs_dbs_data, cpu); 122 dbs_check_cpu(dbs_data, cpu);
127 123
128 schedule_delayed_work_on(smp_processor_id(), dw, delay); 124 schedule_delayed_work_on(smp_processor_id(), dw, delay);
129 mutex_unlock(&core_dbs_info->cdbs.timer_mutex); 125 mutex_unlock(&core_dbs_info->cdbs.timer_mutex);
@@ -154,16 +150,12 @@ static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
154} 150}
155 151
156/************************** sysfs interface ************************/ 152/************************** sysfs interface ************************/
157static ssize_t show_sampling_rate_min(struct kobject *kobj, 153static struct common_dbs_data cs_dbs_cdata;
158 struct attribute *attr, char *buf)
159{
160 return sprintf(buf, "%u\n", cs_dbs_data.min_sampling_rate);
161}
162 154
163static ssize_t store_sampling_down_factor(struct kobject *a, 155static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data,
164 struct attribute *b, 156 const char *buf, size_t count)
165 const char *buf, size_t count)
166{ 157{
158 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
167 unsigned int input; 159 unsigned int input;
168 int ret; 160 int ret;
169 ret = sscanf(buf, "%u", &input); 161 ret = sscanf(buf, "%u", &input);
@@ -171,13 +163,14 @@ static ssize_t store_sampling_down_factor(struct kobject *a,
171 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) 163 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
172 return -EINVAL; 164 return -EINVAL;
173 165
174 cs_tuners.sampling_down_factor = input; 166 cs_tuners->sampling_down_factor = input;
175 return count; 167 return count;
176} 168}
177 169
178static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b, 170static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf,
179 const char *buf, size_t count) 171 size_t count)
180{ 172{
173 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
181 unsigned int input; 174 unsigned int input;
182 int ret; 175 int ret;
183 ret = sscanf(buf, "%u", &input); 176 ret = sscanf(buf, "%u", &input);
@@ -185,43 +178,46 @@ static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b,
185 if (ret != 1) 178 if (ret != 1)
186 return -EINVAL; 179 return -EINVAL;
187 180
188 cs_tuners.sampling_rate = max(input, cs_dbs_data.min_sampling_rate); 181 cs_tuners->sampling_rate = max(input, dbs_data->min_sampling_rate);
189 return count; 182 return count;
190} 183}
191 184
192static ssize_t store_up_threshold(struct kobject *a, struct attribute *b, 185static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf,
193 const char *buf, size_t count) 186 size_t count)
194{ 187{
188 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
195 unsigned int input; 189 unsigned int input;
196 int ret; 190 int ret;
197 ret = sscanf(buf, "%u", &input); 191 ret = sscanf(buf, "%u", &input);
198 192
199 if (ret != 1 || input > 100 || input <= cs_tuners.down_threshold) 193 if (ret != 1 || input > 100 || input <= cs_tuners->down_threshold)
200 return -EINVAL; 194 return -EINVAL;
201 195
202 cs_tuners.up_threshold = input; 196 cs_tuners->up_threshold = input;
203 return count; 197 return count;
204} 198}
205 199
206static ssize_t store_down_threshold(struct kobject *a, struct attribute *b, 200static ssize_t store_down_threshold(struct dbs_data *dbs_data, const char *buf,
207 const char *buf, size_t count) 201 size_t count)
208{ 202{
203 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
209 unsigned int input; 204 unsigned int input;
210 int ret; 205 int ret;
211 ret = sscanf(buf, "%u", &input); 206 ret = sscanf(buf, "%u", &input);
212 207
213 /* cannot be lower than 11 otherwise freq will not fall */ 208 /* cannot be lower than 11 otherwise freq will not fall */
214 if (ret != 1 || input < 11 || input > 100 || 209 if (ret != 1 || input < 11 || input > 100 ||
215 input >= cs_tuners.up_threshold) 210 input >= cs_tuners->up_threshold)
216 return -EINVAL; 211 return -EINVAL;
217 212
218 cs_tuners.down_threshold = input; 213 cs_tuners->down_threshold = input;
219 return count; 214 return count;
220} 215}
221 216
222static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, 217static ssize_t store_ignore_nice(struct dbs_data *dbs_data, const char *buf,
223 const char *buf, size_t count) 218 size_t count)
224{ 219{
220 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
225 unsigned int input, j; 221 unsigned int input, j;
226 int ret; 222 int ret;
227 223
@@ -232,10 +228,10 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b,
232 if (input > 1) 228 if (input > 1)
233 input = 1; 229 input = 1;
234 230
235 if (input == cs_tuners.ignore_nice) /* nothing to do */ 231 if (input == cs_tuners->ignore_nice) /* nothing to do */
236 return count; 232 return count;
237 233
238 cs_tuners.ignore_nice = input; 234 cs_tuners->ignore_nice = input;
239 235
240 /* we need to re-evaluate prev_cpu_idle */ 236 /* we need to re-evaluate prev_cpu_idle */
241 for_each_online_cpu(j) { 237 for_each_online_cpu(j) {
@@ -243,16 +239,17 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b,
243 dbs_info = &per_cpu(cs_cpu_dbs_info, j); 239 dbs_info = &per_cpu(cs_cpu_dbs_info, j);
244 dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j, 240 dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j,
245 &dbs_info->cdbs.prev_cpu_wall); 241 &dbs_info->cdbs.prev_cpu_wall);
246 if (cs_tuners.ignore_nice) 242 if (cs_tuners->ignore_nice)
247 dbs_info->cdbs.prev_cpu_nice = 243 dbs_info->cdbs.prev_cpu_nice =
248 kcpustat_cpu(j).cpustat[CPUTIME_NICE]; 244 kcpustat_cpu(j).cpustat[CPUTIME_NICE];
249 } 245 }
250 return count; 246 return count;
251} 247}
252 248
253static ssize_t store_freq_step(struct kobject *a, struct attribute *b, 249static ssize_t store_freq_step(struct dbs_data *dbs_data, const char *buf,
254 const char *buf, size_t count) 250 size_t count)
255{ 251{
252 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
256 unsigned int input; 253 unsigned int input;
257 int ret; 254 int ret;
258 ret = sscanf(buf, "%u", &input); 255 ret = sscanf(buf, "%u", &input);
@@ -267,43 +264,88 @@ static ssize_t store_freq_step(struct kobject *a, struct attribute *b,
267 * no need to test here if freq_step is zero as the user might actually 264 * no need to test here if freq_step is zero as the user might actually
268 * want this, they would be crazy though :) 265 * want this, they would be crazy though :)
269 */ 266 */
270 cs_tuners.freq_step = input; 267 cs_tuners->freq_step = input;
271 return count; 268 return count;
272} 269}
273 270
274show_one(cs, sampling_rate, sampling_rate); 271show_store_one(cs, sampling_rate);
275show_one(cs, sampling_down_factor, sampling_down_factor); 272show_store_one(cs, sampling_down_factor);
276show_one(cs, up_threshold, up_threshold); 273show_store_one(cs, up_threshold);
277show_one(cs, down_threshold, down_threshold); 274show_store_one(cs, down_threshold);
278show_one(cs, ignore_nice_load, ignore_nice); 275show_store_one(cs, ignore_nice);
279show_one(cs, freq_step, freq_step); 276show_store_one(cs, freq_step);
280 277declare_show_sampling_rate_min(cs);
281define_one_global_rw(sampling_rate); 278
282define_one_global_rw(sampling_down_factor); 279gov_sys_pol_attr_rw(sampling_rate);
283define_one_global_rw(up_threshold); 280gov_sys_pol_attr_rw(sampling_down_factor);
284define_one_global_rw(down_threshold); 281gov_sys_pol_attr_rw(up_threshold);
285define_one_global_rw(ignore_nice_load); 282gov_sys_pol_attr_rw(down_threshold);
286define_one_global_rw(freq_step); 283gov_sys_pol_attr_rw(ignore_nice);
287define_one_global_ro(sampling_rate_min); 284gov_sys_pol_attr_rw(freq_step);
288 285gov_sys_pol_attr_ro(sampling_rate_min);
289static struct attribute *dbs_attributes[] = { 286
290 &sampling_rate_min.attr, 287static struct attribute *dbs_attributes_gov_sys[] = {
291 &sampling_rate.attr, 288 &sampling_rate_min_gov_sys.attr,
292 &sampling_down_factor.attr, 289 &sampling_rate_gov_sys.attr,
293 &up_threshold.attr, 290 &sampling_down_factor_gov_sys.attr,
294 &down_threshold.attr, 291 &up_threshold_gov_sys.attr,
295 &ignore_nice_load.attr, 292 &down_threshold_gov_sys.attr,
296 &freq_step.attr, 293 &ignore_nice_gov_sys.attr,
294 &freq_step_gov_sys.attr,
297 NULL 295 NULL
298}; 296};
299 297
300static struct attribute_group cs_attr_group = { 298static struct attribute_group cs_attr_group_gov_sys = {
301 .attrs = dbs_attributes, 299 .attrs = dbs_attributes_gov_sys,
300 .name = "conservative",
301};
302
303static struct attribute *dbs_attributes_gov_pol[] = {
304 &sampling_rate_min_gov_pol.attr,
305 &sampling_rate_gov_pol.attr,
306 &sampling_down_factor_gov_pol.attr,
307 &up_threshold_gov_pol.attr,
308 &down_threshold_gov_pol.attr,
309 &ignore_nice_gov_pol.attr,
310 &freq_step_gov_pol.attr,
311 NULL
312};
313
314static struct attribute_group cs_attr_group_gov_pol = {
315 .attrs = dbs_attributes_gov_pol,
302 .name = "conservative", 316 .name = "conservative",
303}; 317};
304 318
305/************************** sysfs end ************************/ 319/************************** sysfs end ************************/
306 320
321static int cs_init(struct dbs_data *dbs_data)
322{
323 struct cs_dbs_tuners *tuners;
324
325 tuners = kzalloc(sizeof(struct cs_dbs_tuners), GFP_KERNEL);
326 if (!tuners) {
327 pr_err("%s: kzalloc failed\n", __func__);
328 return -ENOMEM;
329 }
330
331 tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
332 tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD;
333 tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
334 tuners->ignore_nice = 0;
335 tuners->freq_step = 5;
336
337 dbs_data->tuners = tuners;
338 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
339 jiffies_to_usecs(10);
340 mutex_init(&dbs_data->mutex);
341 return 0;
342}
343
344static void cs_exit(struct dbs_data *dbs_data)
345{
346 kfree(dbs_data->tuners);
347}
348
307define_get_cpu_dbs_routines(cs_cpu_dbs_info); 349define_get_cpu_dbs_routines(cs_cpu_dbs_info);
308 350
309static struct notifier_block cs_cpufreq_notifier_block = { 351static struct notifier_block cs_cpufreq_notifier_block = {
@@ -314,21 +356,23 @@ static struct cs_ops cs_ops = {
314 .notifier_block = &cs_cpufreq_notifier_block, 356 .notifier_block = &cs_cpufreq_notifier_block,
315}; 357};
316 358
317static struct dbs_data cs_dbs_data = { 359static struct common_dbs_data cs_dbs_cdata = {
318 .governor = GOV_CONSERVATIVE, 360 .governor = GOV_CONSERVATIVE,
319 .attr_group = &cs_attr_group, 361 .attr_group_gov_sys = &cs_attr_group_gov_sys,
320 .tuners = &cs_tuners, 362 .attr_group_gov_pol = &cs_attr_group_gov_pol,
321 .get_cpu_cdbs = get_cpu_cdbs, 363 .get_cpu_cdbs = get_cpu_cdbs,
322 .get_cpu_dbs_info_s = get_cpu_dbs_info_s, 364 .get_cpu_dbs_info_s = get_cpu_dbs_info_s,
323 .gov_dbs_timer = cs_dbs_timer, 365 .gov_dbs_timer = cs_dbs_timer,
324 .gov_check_cpu = cs_check_cpu, 366 .gov_check_cpu = cs_check_cpu,
325 .gov_ops = &cs_ops, 367 .gov_ops = &cs_ops,
368 .init = cs_init,
369 .exit = cs_exit,
326}; 370};
327 371
328static int cs_cpufreq_governor_dbs(struct cpufreq_policy *policy, 372static int cs_cpufreq_governor_dbs(struct cpufreq_policy *policy,
329 unsigned int event) 373 unsigned int event)
330{ 374{
331 return cpufreq_governor_dbs(&cs_dbs_data, policy, event); 375 return cpufreq_governor_dbs(policy, &cs_dbs_cdata, event);
332} 376}
333 377
334#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE 378#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE
@@ -343,7 +387,6 @@ struct cpufreq_governor cpufreq_gov_conservative = {
343 387
344static int __init cpufreq_gov_dbs_init(void) 388static int __init cpufreq_gov_dbs_init(void)
345{ 389{
346 mutex_init(&cs_dbs_data.mutex);
347 return cpufreq_register_governor(&cpufreq_gov_conservative); 390 return cpufreq_register_governor(&cpufreq_gov_conservative);
348} 391}
349 392
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index 5a76086ff09b..26fbb729bc1c 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -22,12 +22,29 @@
22#include <linux/export.h> 22#include <linux/export.h>
23#include <linux/kernel_stat.h> 23#include <linux/kernel_stat.h>
24#include <linux/mutex.h> 24#include <linux/mutex.h>
25#include <linux/slab.h>
25#include <linux/tick.h> 26#include <linux/tick.h>
26#include <linux/types.h> 27#include <linux/types.h>
27#include <linux/workqueue.h> 28#include <linux/workqueue.h>
28 29
29#include "cpufreq_governor.h" 30#include "cpufreq_governor.h"
30 31
32static struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
33{
34 if (have_governor_per_policy())
35 return &policy->kobj;
36 else
37 return cpufreq_global_kobject;
38}
39
40static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data)
41{
42 if (have_governor_per_policy())
43 return dbs_data->cdata->attr_group_gov_pol;
44 else
45 return dbs_data->cdata->attr_group_gov_sys;
46}
47
31static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) 48static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
32{ 49{
33 u64 idle_time; 50 u64 idle_time;
@@ -65,7 +82,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time);
65 82
66void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) 83void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
67{ 84{
68 struct cpu_dbs_common_info *cdbs = dbs_data->get_cpu_cdbs(cpu); 85 struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
69 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 86 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
70 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 87 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
71 struct cpufreq_policy *policy; 88 struct cpufreq_policy *policy;
@@ -73,7 +90,7 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
73 unsigned int ignore_nice; 90 unsigned int ignore_nice;
74 unsigned int j; 91 unsigned int j;
75 92
76 if (dbs_data->governor == GOV_ONDEMAND) 93 if (dbs_data->cdata->governor == GOV_ONDEMAND)
77 ignore_nice = od_tuners->ignore_nice; 94 ignore_nice = od_tuners->ignore_nice;
78 else 95 else
79 ignore_nice = cs_tuners->ignore_nice; 96 ignore_nice = cs_tuners->ignore_nice;
@@ -87,7 +104,7 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
87 unsigned int idle_time, wall_time, iowait_time; 104 unsigned int idle_time, wall_time, iowait_time;
88 unsigned int load; 105 unsigned int load;
89 106
90 j_cdbs = dbs_data->get_cpu_cdbs(j); 107 j_cdbs = dbs_data->cdata->get_cpu_cdbs(j);
91 108
92 cur_idle_time = get_cpu_idle_time(j, &cur_wall_time); 109 cur_idle_time = get_cpu_idle_time(j, &cur_wall_time);
93 110
@@ -117,9 +134,9 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
117 idle_time += jiffies_to_usecs(cur_nice_jiffies); 134 idle_time += jiffies_to_usecs(cur_nice_jiffies);
118 } 135 }
119 136
120 if (dbs_data->governor == GOV_ONDEMAND) { 137 if (dbs_data->cdata->governor == GOV_ONDEMAND) {
121 struct od_cpu_dbs_info_s *od_j_dbs_info = 138 struct od_cpu_dbs_info_s *od_j_dbs_info =
122 dbs_data->get_cpu_dbs_info_s(cpu); 139 dbs_data->cdata->get_cpu_dbs_info_s(cpu);
123 140
124 cur_iowait_time = get_cpu_iowait_time_us(j, 141 cur_iowait_time = get_cpu_iowait_time_us(j,
125 &cur_wall_time); 142 &cur_wall_time);
@@ -145,7 +162,7 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
145 162
146 load = 100 * (wall_time - idle_time) / wall_time; 163 load = 100 * (wall_time - idle_time) / wall_time;
147 164
148 if (dbs_data->governor == GOV_ONDEMAND) { 165 if (dbs_data->cdata->governor == GOV_ONDEMAND) {
149 int freq_avg = __cpufreq_driver_getavg(policy, j); 166 int freq_avg = __cpufreq_driver_getavg(policy, j);
150 if (freq_avg <= 0) 167 if (freq_avg <= 0)
151 freq_avg = policy->cur; 168 freq_avg = policy->cur;
@@ -157,7 +174,7 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
157 max_load = load; 174 max_load = load;
158 } 175 }
159 176
160 dbs_data->gov_check_cpu(cpu, max_load); 177 dbs_data->cdata->gov_check_cpu(cpu, max_load);
161} 178}
162EXPORT_SYMBOL_GPL(dbs_check_cpu); 179EXPORT_SYMBOL_GPL(dbs_check_cpu);
163 180
@@ -165,14 +182,14 @@ static inline void dbs_timer_init(struct dbs_data *dbs_data, int cpu,
165 unsigned int sampling_rate) 182 unsigned int sampling_rate)
166{ 183{
167 int delay = delay_for_sampling_rate(sampling_rate); 184 int delay = delay_for_sampling_rate(sampling_rate);
168 struct cpu_dbs_common_info *cdbs = dbs_data->get_cpu_cdbs(cpu); 185 struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
169 186
170 schedule_delayed_work_on(cpu, &cdbs->work, delay); 187 schedule_delayed_work_on(cpu, &cdbs->work, delay);
171} 188}
172 189
173static inline void dbs_timer_exit(struct dbs_data *dbs_data, int cpu) 190static inline void dbs_timer_exit(struct dbs_data *dbs_data, int cpu)
174{ 191{
175 struct cpu_dbs_common_info *cdbs = dbs_data->get_cpu_cdbs(cpu); 192 struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
176 193
177 cancel_delayed_work_sync(&cdbs->work); 194 cancel_delayed_work_sync(&cdbs->work);
178} 195}
@@ -196,31 +213,128 @@ bool need_load_eval(struct cpu_dbs_common_info *cdbs,
196} 213}
197EXPORT_SYMBOL_GPL(need_load_eval); 214EXPORT_SYMBOL_GPL(need_load_eval);
198 215
199int cpufreq_governor_dbs(struct dbs_data *dbs_data, 216static void set_sampling_rate(struct dbs_data *dbs_data,
200 struct cpufreq_policy *policy, unsigned int event) 217 unsigned int sampling_rate)
218{
219 if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
220 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
221 cs_tuners->sampling_rate = sampling_rate;
222 } else {
223 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
224 od_tuners->sampling_rate = sampling_rate;
225 }
226}
227
228int cpufreq_governor_dbs(struct cpufreq_policy *policy,
229 struct common_dbs_data *cdata, unsigned int event)
201{ 230{
231 struct dbs_data *dbs_data;
202 struct od_cpu_dbs_info_s *od_dbs_info = NULL; 232 struct od_cpu_dbs_info_s *od_dbs_info = NULL;
203 struct cs_cpu_dbs_info_s *cs_dbs_info = NULL; 233 struct cs_cpu_dbs_info_s *cs_dbs_info = NULL;
204 struct cs_ops *cs_ops = NULL;
205 struct od_ops *od_ops = NULL; 234 struct od_ops *od_ops = NULL;
206 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 235 struct od_dbs_tuners *od_tuners = NULL;
207 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 236 struct cs_dbs_tuners *cs_tuners = NULL;
208 struct cpu_dbs_common_info *cpu_cdbs; 237 struct cpu_dbs_common_info *cpu_cdbs;
209 unsigned int *sampling_rate, latency, ignore_nice, j, cpu = policy->cpu; 238 unsigned int sampling_rate, latency, ignore_nice, j, cpu = policy->cpu;
210 int rc; 239 int rc;
211 240
212 cpu_cdbs = dbs_data->get_cpu_cdbs(cpu); 241 if (have_governor_per_policy())
242 dbs_data = policy->governor_data;
243 else
244 dbs_data = cdata->gdbs_data;
245
246 WARN_ON(!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT));
247
248 switch (event) {
249 case CPUFREQ_GOV_POLICY_INIT:
250 if (have_governor_per_policy()) {
251 WARN_ON(dbs_data);
252 } else if (dbs_data) {
253 policy->governor_data = dbs_data;
254 return 0;
255 }
256
257 dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL);
258 if (!dbs_data) {
259 pr_err("%s: POLICY_INIT: kzalloc failed\n", __func__);
260 return -ENOMEM;
261 }
262
263 dbs_data->cdata = cdata;
264 rc = cdata->init(dbs_data);
265 if (rc) {
266 pr_err("%s: POLICY_INIT: init() failed\n", __func__);
267 kfree(dbs_data);
268 return rc;
269 }
270
271 rc = sysfs_create_group(get_governor_parent_kobj(policy),
272 get_sysfs_attr(dbs_data));
273 if (rc) {
274 cdata->exit(dbs_data);
275 kfree(dbs_data);
276 return rc;
277 }
278
279 policy->governor_data = dbs_data;
280
281 /* policy latency is in nS. Convert it to uS first */
282 latency = policy->cpuinfo.transition_latency / 1000;
283 if (latency == 0)
284 latency = 1;
285
286 /* Bring kernel and HW constraints together */
287 dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate,
288 MIN_LATENCY_MULTIPLIER * latency);
289 set_sampling_rate(dbs_data, max(dbs_data->min_sampling_rate,
290 latency * LATENCY_MULTIPLIER));
291
292 if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
293 struct cs_ops *cs_ops = dbs_data->cdata->gov_ops;
294
295 cpufreq_register_notifier(cs_ops->notifier_block,
296 CPUFREQ_TRANSITION_NOTIFIER);
297 }
298
299 if (!have_governor_per_policy())
300 cdata->gdbs_data = dbs_data;
301
302 return 0;
303 case CPUFREQ_GOV_POLICY_EXIT:
304 if ((policy->governor->initialized == 1) ||
305 have_governor_per_policy()) {
306 sysfs_remove_group(get_governor_parent_kobj(policy),
307 get_sysfs_attr(dbs_data));
308
309 if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
310 struct cs_ops *cs_ops = dbs_data->cdata->gov_ops;
311
312 cpufreq_unregister_notifier(cs_ops->notifier_block,
313 CPUFREQ_TRANSITION_NOTIFIER);
314 }
315
316 cdata->exit(dbs_data);
317 kfree(dbs_data);
318 cdata->gdbs_data = NULL;
319 }
213 320
214 if (dbs_data->governor == GOV_CONSERVATIVE) { 321 policy->governor_data = NULL;
215 cs_dbs_info = dbs_data->get_cpu_dbs_info_s(cpu); 322 return 0;
216 sampling_rate = &cs_tuners->sampling_rate; 323 }
324
325 cpu_cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
326
327 if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
328 cs_tuners = dbs_data->tuners;
329 cs_dbs_info = dbs_data->cdata->get_cpu_dbs_info_s(cpu);
330 sampling_rate = cs_tuners->sampling_rate;
217 ignore_nice = cs_tuners->ignore_nice; 331 ignore_nice = cs_tuners->ignore_nice;
218 cs_ops = dbs_data->gov_ops;
219 } else { 332 } else {
220 od_dbs_info = dbs_data->get_cpu_dbs_info_s(cpu); 333 od_tuners = dbs_data->tuners;
221 sampling_rate = &od_tuners->sampling_rate; 334 od_dbs_info = dbs_data->cdata->get_cpu_dbs_info_s(cpu);
335 sampling_rate = od_tuners->sampling_rate;
222 ignore_nice = od_tuners->ignore_nice; 336 ignore_nice = od_tuners->ignore_nice;
223 od_ops = dbs_data->gov_ops; 337 od_ops = dbs_data->cdata->gov_ops;
224 } 338 }
225 339
226 switch (event) { 340 switch (event) {
@@ -232,7 +346,7 @@ int cpufreq_governor_dbs(struct dbs_data *dbs_data,
232 346
233 for_each_cpu(j, policy->cpus) { 347 for_each_cpu(j, policy->cpus) {
234 struct cpu_dbs_common_info *j_cdbs = 348 struct cpu_dbs_common_info *j_cdbs =
235 dbs_data->get_cpu_cdbs(j); 349 dbs_data->cdata->get_cpu_cdbs(j);
236 350
237 j_cdbs->cpu = j; 351 j_cdbs->cpu = j;
238 j_cdbs->cur_policy = policy; 352 j_cdbs->cur_policy = policy;
@@ -244,69 +358,34 @@ int cpufreq_governor_dbs(struct dbs_data *dbs_data,
244 358
245 mutex_init(&j_cdbs->timer_mutex); 359 mutex_init(&j_cdbs->timer_mutex);
246 INIT_DEFERRABLE_WORK(&j_cdbs->work, 360 INIT_DEFERRABLE_WORK(&j_cdbs->work,
247 dbs_data->gov_dbs_timer); 361 dbs_data->cdata->gov_dbs_timer);
248 }
249
250 if (!policy->governor->initialized) {
251 rc = sysfs_create_group(cpufreq_global_kobject,
252 dbs_data->attr_group);
253 if (rc) {
254 mutex_unlock(&dbs_data->mutex);
255 return rc;
256 }
257 } 362 }
258 363
259 /* 364 /*
260 * conservative does not implement micro like ondemand 365 * conservative does not implement micro like ondemand
261 * governor, thus we are bound to jiffes/HZ 366 * governor, thus we are bound to jiffes/HZ
262 */ 367 */
263 if (dbs_data->governor == GOV_CONSERVATIVE) { 368 if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
264 cs_dbs_info->down_skip = 0; 369 cs_dbs_info->down_skip = 0;
265 cs_dbs_info->enable = 1; 370 cs_dbs_info->enable = 1;
266 cs_dbs_info->requested_freq = policy->cur; 371 cs_dbs_info->requested_freq = policy->cur;
267
268 if (!policy->governor->initialized) {
269 cpufreq_register_notifier(cs_ops->notifier_block,
270 CPUFREQ_TRANSITION_NOTIFIER);
271
272 dbs_data->min_sampling_rate =
273 MIN_SAMPLING_RATE_RATIO *
274 jiffies_to_usecs(10);
275 }
276 } else { 372 } else {
277 od_dbs_info->rate_mult = 1; 373 od_dbs_info->rate_mult = 1;
278 od_dbs_info->sample_type = OD_NORMAL_SAMPLE; 374 od_dbs_info->sample_type = OD_NORMAL_SAMPLE;
279 od_ops->powersave_bias_init_cpu(cpu); 375 od_ops->powersave_bias_init_cpu(cpu);
280
281 if (!policy->governor->initialized)
282 od_tuners->io_is_busy = od_ops->io_busy();
283 } 376 }
284 377
285 if (policy->governor->initialized)
286 goto unlock;
287
288 /* policy latency is in nS. Convert it to uS first */
289 latency = policy->cpuinfo.transition_latency / 1000;
290 if (latency == 0)
291 latency = 1;
292
293 /* Bring kernel and HW constraints together */
294 dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate,
295 MIN_LATENCY_MULTIPLIER * latency);
296 *sampling_rate = max(dbs_data->min_sampling_rate, latency *
297 LATENCY_MULTIPLIER);
298unlock:
299 mutex_unlock(&dbs_data->mutex); 378 mutex_unlock(&dbs_data->mutex);
300 379
301 /* Initiate timer time stamp */ 380 /* Initiate timer time stamp */
302 cpu_cdbs->time_stamp = ktime_get(); 381 cpu_cdbs->time_stamp = ktime_get();
303 382
304 for_each_cpu(j, policy->cpus) 383 for_each_cpu(j, policy->cpus)
305 dbs_timer_init(dbs_data, j, *sampling_rate); 384 dbs_timer_init(dbs_data, j, sampling_rate);
306 break; 385 break;
307 386
308 case CPUFREQ_GOV_STOP: 387 case CPUFREQ_GOV_STOP:
309 if (dbs_data->governor == GOV_CONSERVATIVE) 388 if (dbs_data->cdata->governor == GOV_CONSERVATIVE)
310 cs_dbs_info->enable = 0; 389 cs_dbs_info->enable = 0;
311 390
312 for_each_cpu(j, policy->cpus) 391 for_each_cpu(j, policy->cpus)
@@ -315,13 +394,6 @@ unlock:
315 mutex_lock(&dbs_data->mutex); 394 mutex_lock(&dbs_data->mutex);
316 mutex_destroy(&cpu_cdbs->timer_mutex); 395 mutex_destroy(&cpu_cdbs->timer_mutex);
317 396
318 if (policy->governor->initialized == 1) {
319 sysfs_remove_group(cpufreq_global_kobject,
320 dbs_data->attr_group);
321 if (dbs_data->governor == GOV_CONSERVATIVE)
322 cpufreq_unregister_notifier(cs_ops->notifier_block,
323 CPUFREQ_TRANSITION_NOTIFIER);
324 }
325 mutex_unlock(&dbs_data->mutex); 397 mutex_unlock(&dbs_data->mutex);
326 398
327 break; 399 break;
diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h
index 46bde01eee62..c83cabf14b2f 100644
--- a/drivers/cpufreq/cpufreq_governor.h
+++ b/drivers/cpufreq/cpufreq_governor.h
@@ -40,14 +40,75 @@
40/* Ondemand Sampling types */ 40/* Ondemand Sampling types */
41enum {OD_NORMAL_SAMPLE, OD_SUB_SAMPLE}; 41enum {OD_NORMAL_SAMPLE, OD_SUB_SAMPLE};
42 42
43/* Macro creating sysfs show routines */ 43/*
44#define show_one(_gov, file_name, object) \ 44 * Macro for creating governors sysfs routines
45static ssize_t show_##file_name \ 45 *
46 * - gov_sys: One governor instance per whole system
47 * - gov_pol: One governor instance per policy
48 */
49
50/* Create attributes */
51#define gov_sys_attr_ro(_name) \
52static struct global_attr _name##_gov_sys = \
53__ATTR(_name, 0444, show_##_name##_gov_sys, NULL)
54
55#define gov_sys_attr_rw(_name) \
56static struct global_attr _name##_gov_sys = \
57__ATTR(_name, 0644, show_##_name##_gov_sys, store_##_name##_gov_sys)
58
59#define gov_pol_attr_ro(_name) \
60static struct freq_attr _name##_gov_pol = \
61__ATTR(_name, 0444, show_##_name##_gov_pol, NULL)
62
63#define gov_pol_attr_rw(_name) \
64static struct freq_attr _name##_gov_pol = \
65__ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol)
66
67#define gov_sys_pol_attr_rw(_name) \
68 gov_sys_attr_rw(_name); \
69 gov_pol_attr_rw(_name)
70
71#define gov_sys_pol_attr_ro(_name) \
72 gov_sys_attr_ro(_name); \
73 gov_pol_attr_ro(_name)
74
75/* Create show/store routines */
76#define show_one(_gov, file_name) \
77static ssize_t show_##file_name##_gov_sys \
46(struct kobject *kobj, struct attribute *attr, char *buf) \ 78(struct kobject *kobj, struct attribute *attr, char *buf) \
47{ \ 79{ \
48 return sprintf(buf, "%u\n", _gov##_tuners.object); \ 80 struct _gov##_dbs_tuners *tuners = _gov##_dbs_cdata.gdbs_data->tuners; \
81 return sprintf(buf, "%u\n", tuners->file_name); \
82} \
83 \
84static ssize_t show_##file_name##_gov_pol \
85(struct cpufreq_policy *policy, char *buf) \
86{ \
87 struct dbs_data *dbs_data = policy->governor_data; \
88 struct _gov##_dbs_tuners *tuners = dbs_data->tuners; \
89 return sprintf(buf, "%u\n", tuners->file_name); \
90}
91
92#define store_one(_gov, file_name) \
93static ssize_t store_##file_name##_gov_sys \
94(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) \
95{ \
96 struct dbs_data *dbs_data = _gov##_dbs_cdata.gdbs_data; \
97 return store_##file_name(dbs_data, buf, count); \
98} \
99 \
100static ssize_t store_##file_name##_gov_pol \
101(struct cpufreq_policy *policy, const char *buf, size_t count) \
102{ \
103 struct dbs_data *dbs_data = policy->governor_data; \
104 return store_##file_name(dbs_data, buf, count); \
49} 105}
50 106
107#define show_store_one(_gov, file_name) \
108show_one(_gov, file_name); \
109store_one(_gov, file_name)
110
111/* create helper routines */
51#define define_get_cpu_dbs_routines(_dbs_info) \ 112#define define_get_cpu_dbs_routines(_dbs_info) \
52static struct cpu_dbs_common_info *get_cpu_cdbs(int cpu) \ 113static struct cpu_dbs_common_info *get_cpu_cdbs(int cpu) \
53{ \ 114{ \
@@ -103,7 +164,7 @@ struct cs_cpu_dbs_info_s {
103 unsigned int enable:1; 164 unsigned int enable:1;
104}; 165};
105 166
106/* Governers sysfs tunables */ 167/* Per policy Governers sysfs tunables */
107struct od_dbs_tuners { 168struct od_dbs_tuners {
108 unsigned int ignore_nice; 169 unsigned int ignore_nice;
109 unsigned int sampling_rate; 170 unsigned int sampling_rate;
@@ -123,31 +184,42 @@ struct cs_dbs_tuners {
123 unsigned int freq_step; 184 unsigned int freq_step;
124}; 185};
125 186
126/* Per Governer data */ 187/* Common Governer data across policies */
127struct dbs_data { 188struct dbs_data;
189struct common_dbs_data {
128 /* Common across governors */ 190 /* Common across governors */
129 #define GOV_ONDEMAND 0 191 #define GOV_ONDEMAND 0
130 #define GOV_CONSERVATIVE 1 192 #define GOV_CONSERVATIVE 1
131 int governor; 193 int governor;
132 unsigned int min_sampling_rate; 194 struct attribute_group *attr_group_gov_sys; /* one governor - system */
133 struct attribute_group *attr_group; 195 struct attribute_group *attr_group_gov_pol; /* one governor - policy */
134 void *tuners;
135 196
136 /* dbs_mutex protects dbs_enable in governor start/stop */ 197 /* Common data for platforms that don't set have_governor_per_policy */
137 struct mutex mutex; 198 struct dbs_data *gdbs_data;
138 199
139 struct cpu_dbs_common_info *(*get_cpu_cdbs)(int cpu); 200 struct cpu_dbs_common_info *(*get_cpu_cdbs)(int cpu);
140 void *(*get_cpu_dbs_info_s)(int cpu); 201 void *(*get_cpu_dbs_info_s)(int cpu);
141 void (*gov_dbs_timer)(struct work_struct *work); 202 void (*gov_dbs_timer)(struct work_struct *work);
142 void (*gov_check_cpu)(int cpu, unsigned int load); 203 void (*gov_check_cpu)(int cpu, unsigned int load);
204 int (*init)(struct dbs_data *dbs_data);
205 void (*exit)(struct dbs_data *dbs_data);
143 206
144 /* Governor specific ops, see below */ 207 /* Governor specific ops, see below */
145 void *gov_ops; 208 void *gov_ops;
146}; 209};
147 210
211/* Governer Per policy data */
212struct dbs_data {
213 struct common_dbs_data *cdata;
214 unsigned int min_sampling_rate;
215 void *tuners;
216
217 /* dbs_mutex protects dbs_enable in governor start/stop */
218 struct mutex mutex;
219};
220
148/* Governor specific ops, will be passed to dbs_data->gov_ops */ 221/* Governor specific ops, will be passed to dbs_data->gov_ops */
149struct od_ops { 222struct od_ops {
150 int (*io_busy)(void);
151 void (*powersave_bias_init_cpu)(int cpu); 223 void (*powersave_bias_init_cpu)(int cpu);
152 unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy, 224 unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy,
153 unsigned int freq_next, unsigned int relation); 225 unsigned int freq_next, unsigned int relation);
@@ -169,10 +241,25 @@ static inline int delay_for_sampling_rate(unsigned int sampling_rate)
169 return delay; 241 return delay;
170} 242}
171 243
244#define declare_show_sampling_rate_min(_gov) \
245static ssize_t show_sampling_rate_min_gov_sys \
246(struct kobject *kobj, struct attribute *attr, char *buf) \
247{ \
248 struct dbs_data *dbs_data = _gov##_dbs_cdata.gdbs_data; \
249 return sprintf(buf, "%u\n", dbs_data->min_sampling_rate); \
250} \
251 \
252static ssize_t show_sampling_rate_min_gov_pol \
253(struct cpufreq_policy *policy, char *buf) \
254{ \
255 struct dbs_data *dbs_data = policy->governor_data; \
256 return sprintf(buf, "%u\n", dbs_data->min_sampling_rate); \
257}
258
172u64 get_cpu_idle_time(unsigned int cpu, u64 *wall); 259u64 get_cpu_idle_time(unsigned int cpu, u64 *wall);
173void dbs_check_cpu(struct dbs_data *dbs_data, int cpu); 260void dbs_check_cpu(struct dbs_data *dbs_data, int cpu);
174bool need_load_eval(struct cpu_dbs_common_info *cdbs, 261bool need_load_eval(struct cpu_dbs_common_info *cdbs,
175 unsigned int sampling_rate); 262 unsigned int sampling_rate);
176int cpufreq_governor_dbs(struct dbs_data *dbs_data, 263int cpufreq_governor_dbs(struct cpufreq_policy *policy,
177 struct cpufreq_policy *policy, unsigned int event); 264 struct common_dbs_data *cdata, unsigned int event);
178#endif /* _CPUFREQ_GOVERNER_H */ 265#endif /* _CPUFREQ_GOVERNER_H */
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
index f3eb26cd848f..15e80ee61352 100644
--- a/drivers/cpufreq/cpufreq_ondemand.c
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -20,6 +20,7 @@
20#include <linux/module.h> 20#include <linux/module.h>
21#include <linux/mutex.h> 21#include <linux/mutex.h>
22#include <linux/percpu-defs.h> 22#include <linux/percpu-defs.h>
23#include <linux/slab.h>
23#include <linux/sysfs.h> 24#include <linux/sysfs.h>
24#include <linux/tick.h> 25#include <linux/tick.h>
25#include <linux/types.h> 26#include <linux/types.h>
@@ -37,22 +38,12 @@
37#define MIN_FREQUENCY_UP_THRESHOLD (11) 38#define MIN_FREQUENCY_UP_THRESHOLD (11)
38#define MAX_FREQUENCY_UP_THRESHOLD (100) 39#define MAX_FREQUENCY_UP_THRESHOLD (100)
39 40
40static struct dbs_data od_dbs_data;
41static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info); 41static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info);
42 42
43#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND 43#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
44static struct cpufreq_governor cpufreq_gov_ondemand; 44static struct cpufreq_governor cpufreq_gov_ondemand;
45#endif 45#endif
46 46
47static struct od_dbs_tuners od_tuners = {
48 .up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
49 .sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR,
50 .adj_up_threshold = DEF_FREQUENCY_UP_THRESHOLD -
51 DEF_FREQUENCY_DOWN_DIFFERENTIAL,
52 .ignore_nice = 0,
53 .powersave_bias = 0,
54};
55
56static void ondemand_powersave_bias_init_cpu(int cpu) 47static void ondemand_powersave_bias_init_cpu(int cpu)
57{ 48{
58 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); 49 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
@@ -98,6 +89,8 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
98 unsigned int jiffies_total, jiffies_hi, jiffies_lo; 89 unsigned int jiffies_total, jiffies_hi, jiffies_lo;
99 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, 90 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
100 policy->cpu); 91 policy->cpu);
92 struct dbs_data *dbs_data = policy->governor_data;
93 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
101 94
102 if (!dbs_info->freq_table) { 95 if (!dbs_info->freq_table) {
103 dbs_info->freq_lo = 0; 96 dbs_info->freq_lo = 0;
@@ -108,7 +101,7 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
108 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next, 101 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
109 relation, &index); 102 relation, &index);
110 freq_req = dbs_info->freq_table[index].frequency; 103 freq_req = dbs_info->freq_table[index].frequency;
111 freq_reduc = freq_req * od_tuners.powersave_bias / 1000; 104 freq_reduc = freq_req * od_tuners->powersave_bias / 1000;
112 freq_avg = freq_req - freq_reduc; 105 freq_avg = freq_req - freq_reduc;
113 106
114 /* Find freq bounds for freq_avg in freq_table */ 107 /* Find freq bounds for freq_avg in freq_table */
@@ -127,7 +120,7 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
127 dbs_info->freq_lo_jiffies = 0; 120 dbs_info->freq_lo_jiffies = 0;
128 return freq_lo; 121 return freq_lo;
129 } 122 }
130 jiffies_total = usecs_to_jiffies(od_tuners.sampling_rate); 123 jiffies_total = usecs_to_jiffies(od_tuners->sampling_rate);
131 jiffies_hi = (freq_avg - freq_lo) * jiffies_total; 124 jiffies_hi = (freq_avg - freq_lo) * jiffies_total;
132 jiffies_hi += ((freq_hi - freq_lo) / 2); 125 jiffies_hi += ((freq_hi - freq_lo) / 2);
133 jiffies_hi /= (freq_hi - freq_lo); 126 jiffies_hi /= (freq_hi - freq_lo);
@@ -148,12 +141,15 @@ static void ondemand_powersave_bias_init(void)
148 141
149static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq) 142static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq)
150{ 143{
151 if (od_tuners.powersave_bias) 144 struct dbs_data *dbs_data = p->governor_data;
145 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
146
147 if (od_tuners->powersave_bias)
152 freq = powersave_bias_target(p, freq, CPUFREQ_RELATION_H); 148 freq = powersave_bias_target(p, freq, CPUFREQ_RELATION_H);
153 else if (p->cur == p->max) 149 else if (p->cur == p->max)
154 return; 150 return;
155 151
156 __cpufreq_driver_target(p, freq, od_tuners.powersave_bias ? 152 __cpufreq_driver_target(p, freq, od_tuners->powersave_bias ?
157 CPUFREQ_RELATION_L : CPUFREQ_RELATION_H); 153 CPUFREQ_RELATION_L : CPUFREQ_RELATION_H);
158} 154}
159 155
@@ -170,15 +166,17 @@ static void od_check_cpu(int cpu, unsigned int load_freq)
170{ 166{
171 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); 167 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
172 struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; 168 struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy;
169 struct dbs_data *dbs_data = policy->governor_data;
170 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
173 171
174 dbs_info->freq_lo = 0; 172 dbs_info->freq_lo = 0;
175 173
176 /* Check for frequency increase */ 174 /* Check for frequency increase */
177 if (load_freq > od_tuners.up_threshold * policy->cur) { 175 if (load_freq > od_tuners->up_threshold * policy->cur) {
178 /* If switching to max speed, apply sampling_down_factor */ 176 /* If switching to max speed, apply sampling_down_factor */
179 if (policy->cur < policy->max) 177 if (policy->cur < policy->max)
180 dbs_info->rate_mult = 178 dbs_info->rate_mult =
181 od_tuners.sampling_down_factor; 179 od_tuners->sampling_down_factor;
182 dbs_freq_increase(policy, policy->max); 180 dbs_freq_increase(policy, policy->max);
183 return; 181 return;
184 } 182 }
@@ -193,9 +191,10 @@ static void od_check_cpu(int cpu, unsigned int load_freq)
193 * support the current CPU usage without triggering the up policy. To be 191 * support the current CPU usage without triggering the up policy. To be
194 * safe, we focus 10 points under the threshold. 192 * safe, we focus 10 points under the threshold.
195 */ 193 */
196 if (load_freq < od_tuners.adj_up_threshold * policy->cur) { 194 if (load_freq < od_tuners->adj_up_threshold
195 * policy->cur) {
197 unsigned int freq_next; 196 unsigned int freq_next;
198 freq_next = load_freq / od_tuners.adj_up_threshold; 197 freq_next = load_freq / od_tuners->adj_up_threshold;
199 198
200 /* No longer fully busy, reset rate_mult */ 199 /* No longer fully busy, reset rate_mult */
201 dbs_info->rate_mult = 1; 200 dbs_info->rate_mult = 1;
@@ -203,7 +202,7 @@ static void od_check_cpu(int cpu, unsigned int load_freq)
203 if (freq_next < policy->min) 202 if (freq_next < policy->min)
204 freq_next = policy->min; 203 freq_next = policy->min;
205 204
206 if (!od_tuners.powersave_bias) { 205 if (!od_tuners->powersave_bias) {
207 __cpufreq_driver_target(policy, freq_next, 206 __cpufreq_driver_target(policy, freq_next,
208 CPUFREQ_RELATION_L); 207 CPUFREQ_RELATION_L);
209 } else { 208 } else {
@@ -223,12 +222,14 @@ static void od_dbs_timer(struct work_struct *work)
223 unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; 222 unsigned int cpu = dbs_info->cdbs.cur_policy->cpu;
224 struct od_cpu_dbs_info_s *core_dbs_info = &per_cpu(od_cpu_dbs_info, 223 struct od_cpu_dbs_info_s *core_dbs_info = &per_cpu(od_cpu_dbs_info,
225 cpu); 224 cpu);
225 struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data;
226 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
226 int delay, sample_type = core_dbs_info->sample_type; 227 int delay, sample_type = core_dbs_info->sample_type;
227 bool eval_load; 228 bool eval_load;
228 229
229 mutex_lock(&core_dbs_info->cdbs.timer_mutex); 230 mutex_lock(&core_dbs_info->cdbs.timer_mutex);
230 eval_load = need_load_eval(&core_dbs_info->cdbs, 231 eval_load = need_load_eval(&core_dbs_info->cdbs,
231 od_tuners.sampling_rate); 232 od_tuners->sampling_rate);
232 233
233 /* Common NORMAL_SAMPLE setup */ 234 /* Common NORMAL_SAMPLE setup */
234 core_dbs_info->sample_type = OD_NORMAL_SAMPLE; 235 core_dbs_info->sample_type = OD_NORMAL_SAMPLE;
@@ -240,13 +241,13 @@ static void od_dbs_timer(struct work_struct *work)
240 CPUFREQ_RELATION_H); 241 CPUFREQ_RELATION_H);
241 } else { 242 } else {
242 if (eval_load) 243 if (eval_load)
243 dbs_check_cpu(&od_dbs_data, cpu); 244 dbs_check_cpu(dbs_data, cpu);
244 if (core_dbs_info->freq_lo) { 245 if (core_dbs_info->freq_lo) {
245 /* Setup timer for SUB_SAMPLE */ 246 /* Setup timer for SUB_SAMPLE */
246 core_dbs_info->sample_type = OD_SUB_SAMPLE; 247 core_dbs_info->sample_type = OD_SUB_SAMPLE;
247 delay = core_dbs_info->freq_hi_jiffies; 248 delay = core_dbs_info->freq_hi_jiffies;
248 } else { 249 } else {
249 delay = delay_for_sampling_rate(od_tuners.sampling_rate 250 delay = delay_for_sampling_rate(od_tuners->sampling_rate
250 * core_dbs_info->rate_mult); 251 * core_dbs_info->rate_mult);
251 } 252 }
252 } 253 }
@@ -256,12 +257,7 @@ static void od_dbs_timer(struct work_struct *work)
256} 257}
257 258
258/************************** sysfs interface ************************/ 259/************************** sysfs interface ************************/
259 260static struct common_dbs_data od_dbs_cdata;
260static ssize_t show_sampling_rate_min(struct kobject *kobj,
261 struct attribute *attr, char *buf)
262{
263 return sprintf(buf, "%u\n", od_dbs_data.min_sampling_rate);
264}
265 261
266/** 262/**
267 * update_sampling_rate - update sampling rate effective immediately if needed. 263 * update_sampling_rate - update sampling rate effective immediately if needed.
@@ -276,12 +272,14 @@ static ssize_t show_sampling_rate_min(struct kobject *kobj,
276 * reducing the sampling rate, we need to make the new value effective 272 * reducing the sampling rate, we need to make the new value effective
277 * immediately. 273 * immediately.
278 */ 274 */
279static void update_sampling_rate(unsigned int new_rate) 275static void update_sampling_rate(struct dbs_data *dbs_data,
276 unsigned int new_rate)
280{ 277{
278 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
281 int cpu; 279 int cpu;
282 280
283 od_tuners.sampling_rate = new_rate = max(new_rate, 281 od_tuners->sampling_rate = new_rate = max(new_rate,
284 od_dbs_data.min_sampling_rate); 282 dbs_data->min_sampling_rate);
285 283
286 for_each_online_cpu(cpu) { 284 for_each_online_cpu(cpu) {
287 struct cpufreq_policy *policy; 285 struct cpufreq_policy *policy;
@@ -322,34 +320,37 @@ static void update_sampling_rate(unsigned int new_rate)
322 } 320 }
323} 321}
324 322
325static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b, 323static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf,
326 const char *buf, size_t count) 324 size_t count)
327{ 325{
328 unsigned int input; 326 unsigned int input;
329 int ret; 327 int ret;
330 ret = sscanf(buf, "%u", &input); 328 ret = sscanf(buf, "%u", &input);
331 if (ret != 1) 329 if (ret != 1)
332 return -EINVAL; 330 return -EINVAL;
333 update_sampling_rate(input); 331
332 update_sampling_rate(dbs_data, input);
334 return count; 333 return count;
335} 334}
336 335
337static ssize_t store_io_is_busy(struct kobject *a, struct attribute *b, 336static ssize_t store_io_is_busy(struct dbs_data *dbs_data, const char *buf,
338 const char *buf, size_t count) 337 size_t count)
339{ 338{
339 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
340 unsigned int input; 340 unsigned int input;
341 int ret; 341 int ret;
342 342
343 ret = sscanf(buf, "%u", &input); 343 ret = sscanf(buf, "%u", &input);
344 if (ret != 1) 344 if (ret != 1)
345 return -EINVAL; 345 return -EINVAL;
346 od_tuners.io_is_busy = !!input; 346 od_tuners->io_is_busy = !!input;
347 return count; 347 return count;
348} 348}
349 349
350static ssize_t store_up_threshold(struct kobject *a, struct attribute *b, 350static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf,
351 const char *buf, size_t count) 351 size_t count)
352{ 352{
353 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
353 unsigned int input; 354 unsigned int input;
354 int ret; 355 int ret;
355 ret = sscanf(buf, "%u", &input); 356 ret = sscanf(buf, "%u", &input);
@@ -359,23 +360,24 @@ static ssize_t store_up_threshold(struct kobject *a, struct attribute *b,
359 return -EINVAL; 360 return -EINVAL;
360 } 361 }
361 /* Calculate the new adj_up_threshold */ 362 /* Calculate the new adj_up_threshold */
362 od_tuners.adj_up_threshold += input; 363 od_tuners->adj_up_threshold += input;
363 od_tuners.adj_up_threshold -= od_tuners.up_threshold; 364 od_tuners->adj_up_threshold -= od_tuners->up_threshold;
364 365
365 od_tuners.up_threshold = input; 366 od_tuners->up_threshold = input;
366 return count; 367 return count;
367} 368}
368 369
369static ssize_t store_sampling_down_factor(struct kobject *a, 370static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data,
370 struct attribute *b, const char *buf, size_t count) 371 const char *buf, size_t count)
371{ 372{
373 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
372 unsigned int input, j; 374 unsigned int input, j;
373 int ret; 375 int ret;
374 ret = sscanf(buf, "%u", &input); 376 ret = sscanf(buf, "%u", &input);
375 377
376 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) 378 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
377 return -EINVAL; 379 return -EINVAL;
378 od_tuners.sampling_down_factor = input; 380 od_tuners->sampling_down_factor = input;
379 381
380 /* Reset down sampling multiplier in case it was active */ 382 /* Reset down sampling multiplier in case it was active */
381 for_each_online_cpu(j) { 383 for_each_online_cpu(j) {
@@ -386,9 +388,10 @@ static ssize_t store_sampling_down_factor(struct kobject *a,
386 return count; 388 return count;
387} 389}
388 390
389static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, 391static ssize_t store_ignore_nice(struct dbs_data *dbs_data, const char *buf,
390 const char *buf, size_t count) 392 size_t count)
391{ 393{
394 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
392 unsigned int input; 395 unsigned int input;
393 int ret; 396 int ret;
394 397
@@ -401,10 +404,10 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b,
401 if (input > 1) 404 if (input > 1)
402 input = 1; 405 input = 1;
403 406
404 if (input == od_tuners.ignore_nice) { /* nothing to do */ 407 if (input == od_tuners->ignore_nice) { /* nothing to do */
405 return count; 408 return count;
406 } 409 }
407 od_tuners.ignore_nice = input; 410 od_tuners->ignore_nice = input;
408 411
409 /* we need to re-evaluate prev_cpu_idle */ 412 /* we need to re-evaluate prev_cpu_idle */
410 for_each_online_cpu(j) { 413 for_each_online_cpu(j) {
@@ -412,7 +415,7 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b,
412 dbs_info = &per_cpu(od_cpu_dbs_info, j); 415 dbs_info = &per_cpu(od_cpu_dbs_info, j);
413 dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j, 416 dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j,
414 &dbs_info->cdbs.prev_cpu_wall); 417 &dbs_info->cdbs.prev_cpu_wall);
415 if (od_tuners.ignore_nice) 418 if (od_tuners->ignore_nice)
416 dbs_info->cdbs.prev_cpu_nice = 419 dbs_info->cdbs.prev_cpu_nice =
417 kcpustat_cpu(j).cpustat[CPUTIME_NICE]; 420 kcpustat_cpu(j).cpustat[CPUTIME_NICE];
418 421
@@ -420,9 +423,10 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b,
420 return count; 423 return count;
421} 424}
422 425
423static ssize_t store_powersave_bias(struct kobject *a, struct attribute *b, 426static ssize_t store_powersave_bias(struct dbs_data *dbs_data, const char *buf,
424 const char *buf, size_t count) 427 size_t count)
425{ 428{
429 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
426 unsigned int input; 430 unsigned int input;
427 int ret; 431 int ret;
428 ret = sscanf(buf, "%u", &input); 432 ret = sscanf(buf, "%u", &input);
@@ -433,68 +437,138 @@ static ssize_t store_powersave_bias(struct kobject *a, struct attribute *b,
433 if (input > 1000) 437 if (input > 1000)
434 input = 1000; 438 input = 1000;
435 439
436 od_tuners.powersave_bias = input; 440 od_tuners->powersave_bias = input;
437 ondemand_powersave_bias_init(); 441 ondemand_powersave_bias_init();
438 return count; 442 return count;
439} 443}
440 444
441show_one(od, sampling_rate, sampling_rate); 445show_store_one(od, sampling_rate);
442show_one(od, io_is_busy, io_is_busy); 446show_store_one(od, io_is_busy);
443show_one(od, up_threshold, up_threshold); 447show_store_one(od, up_threshold);
444show_one(od, sampling_down_factor, sampling_down_factor); 448show_store_one(od, sampling_down_factor);
445show_one(od, ignore_nice_load, ignore_nice); 449show_store_one(od, ignore_nice);
446show_one(od, powersave_bias, powersave_bias); 450show_store_one(od, powersave_bias);
447 451declare_show_sampling_rate_min(od);
448define_one_global_rw(sampling_rate); 452
449define_one_global_rw(io_is_busy); 453gov_sys_pol_attr_rw(sampling_rate);
450define_one_global_rw(up_threshold); 454gov_sys_pol_attr_rw(io_is_busy);
451define_one_global_rw(sampling_down_factor); 455gov_sys_pol_attr_rw(up_threshold);
452define_one_global_rw(ignore_nice_load); 456gov_sys_pol_attr_rw(sampling_down_factor);
453define_one_global_rw(powersave_bias); 457gov_sys_pol_attr_rw(ignore_nice);
454define_one_global_ro(sampling_rate_min); 458gov_sys_pol_attr_rw(powersave_bias);
455 459gov_sys_pol_attr_ro(sampling_rate_min);
456static struct attribute *dbs_attributes[] = { 460
457 &sampling_rate_min.attr, 461static struct attribute *dbs_attributes_gov_sys[] = {
458 &sampling_rate.attr, 462 &sampling_rate_min_gov_sys.attr,
459 &up_threshold.attr, 463 &sampling_rate_gov_sys.attr,
460 &sampling_down_factor.attr, 464 &up_threshold_gov_sys.attr,
461 &ignore_nice_load.attr, 465 &sampling_down_factor_gov_sys.attr,
462 &powersave_bias.attr, 466 &ignore_nice_gov_sys.attr,
463 &io_is_busy.attr, 467 &powersave_bias_gov_sys.attr,
468 &io_is_busy_gov_sys.attr,
464 NULL 469 NULL
465}; 470};
466 471
467static struct attribute_group od_attr_group = { 472static struct attribute_group od_attr_group_gov_sys = {
468 .attrs = dbs_attributes, 473 .attrs = dbs_attributes_gov_sys,
474 .name = "ondemand",
475};
476
477static struct attribute *dbs_attributes_gov_pol[] = {
478 &sampling_rate_min_gov_pol.attr,
479 &sampling_rate_gov_pol.attr,
480 &up_threshold_gov_pol.attr,
481 &sampling_down_factor_gov_pol.attr,
482 &ignore_nice_gov_pol.attr,
483 &powersave_bias_gov_pol.attr,
484 &io_is_busy_gov_pol.attr,
485 NULL
486};
487
488static struct attribute_group od_attr_group_gov_pol = {
489 .attrs = dbs_attributes_gov_pol,
469 .name = "ondemand", 490 .name = "ondemand",
470}; 491};
471 492
472/************************** sysfs end ************************/ 493/************************** sysfs end ************************/
473 494
495static int od_init(struct dbs_data *dbs_data)
496{
497 struct od_dbs_tuners *tuners;
498 u64 idle_time;
499 int cpu;
500
501 tuners = kzalloc(sizeof(struct od_dbs_tuners), GFP_KERNEL);
502 if (!tuners) {
503 pr_err("%s: kzalloc failed\n", __func__);
504 return -ENOMEM;
505 }
506
507 cpu = get_cpu();
508 idle_time = get_cpu_idle_time_us(cpu, NULL);
509 put_cpu();
510 if (idle_time != -1ULL) {
511 /* Idle micro accounting is supported. Use finer thresholds */
512 tuners->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
513 tuners->adj_up_threshold = MICRO_FREQUENCY_UP_THRESHOLD -
514 MICRO_FREQUENCY_DOWN_DIFFERENTIAL;
515 /*
516 * In nohz/micro accounting case we set the minimum frequency
517 * not depending on HZ, but fixed (very low). The deferred
518 * timer might skip some samples if idle/sleeping as needed.
519 */
520 dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
521 } else {
522 tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
523 tuners->adj_up_threshold = DEF_FREQUENCY_UP_THRESHOLD -
524 DEF_FREQUENCY_DOWN_DIFFERENTIAL;
525
526 /* For correct statistics, we need 10 ticks for each measure */
527 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
528 jiffies_to_usecs(10);
529 }
530
531 tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
532 tuners->ignore_nice = 0;
533 tuners->powersave_bias = 0;
534 tuners->io_is_busy = should_io_be_busy();
535
536 dbs_data->tuners = tuners;
537 pr_info("%s: tuners %p\n", __func__, tuners);
538 mutex_init(&dbs_data->mutex);
539 return 0;
540}
541
542static void od_exit(struct dbs_data *dbs_data)
543{
544 kfree(dbs_data->tuners);
545}
546
474define_get_cpu_dbs_routines(od_cpu_dbs_info); 547define_get_cpu_dbs_routines(od_cpu_dbs_info);
475 548
476static struct od_ops od_ops = { 549static struct od_ops od_ops = {
477 .io_busy = should_io_be_busy,
478 .powersave_bias_init_cpu = ondemand_powersave_bias_init_cpu, 550 .powersave_bias_init_cpu = ondemand_powersave_bias_init_cpu,
479 .powersave_bias_target = powersave_bias_target, 551 .powersave_bias_target = powersave_bias_target,
480 .freq_increase = dbs_freq_increase, 552 .freq_increase = dbs_freq_increase,
481}; 553};
482 554
483static struct dbs_data od_dbs_data = { 555static struct common_dbs_data od_dbs_cdata = {
484 .governor = GOV_ONDEMAND, 556 .governor = GOV_ONDEMAND,
485 .attr_group = &od_attr_group, 557 .attr_group_gov_sys = &od_attr_group_gov_sys,
486 .tuners = &od_tuners, 558 .attr_group_gov_pol = &od_attr_group_gov_pol,
487 .get_cpu_cdbs = get_cpu_cdbs, 559 .get_cpu_cdbs = get_cpu_cdbs,
488 .get_cpu_dbs_info_s = get_cpu_dbs_info_s, 560 .get_cpu_dbs_info_s = get_cpu_dbs_info_s,
489 .gov_dbs_timer = od_dbs_timer, 561 .gov_dbs_timer = od_dbs_timer,
490 .gov_check_cpu = od_check_cpu, 562 .gov_check_cpu = od_check_cpu,
491 .gov_ops = &od_ops, 563 .gov_ops = &od_ops,
564 .init = od_init,
565 .exit = od_exit,
492}; 566};
493 567
494static int od_cpufreq_governor_dbs(struct cpufreq_policy *policy, 568static int od_cpufreq_governor_dbs(struct cpufreq_policy *policy,
495 unsigned int event) 569 unsigned int event)
496{ 570{
497 return cpufreq_governor_dbs(&od_dbs_data, policy, event); 571 return cpufreq_governor_dbs(policy, &od_dbs_cdata, event);
498} 572}
499 573
500#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND 574#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
@@ -509,29 +583,6 @@ struct cpufreq_governor cpufreq_gov_ondemand = {
509 583
510static int __init cpufreq_gov_dbs_init(void) 584static int __init cpufreq_gov_dbs_init(void)
511{ 585{
512 u64 idle_time;
513 int cpu = get_cpu();
514
515 mutex_init(&od_dbs_data.mutex);
516 idle_time = get_cpu_idle_time_us(cpu, NULL);
517 put_cpu();
518 if (idle_time != -1ULL) {
519 /* Idle micro accounting is supported. Use finer thresholds */
520 od_tuners.up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
521 od_tuners.adj_up_threshold = MICRO_FREQUENCY_UP_THRESHOLD -
522 MICRO_FREQUENCY_DOWN_DIFFERENTIAL;
523 /*
524 * In nohz/micro accounting case we set the minimum frequency
525 * not depending on HZ, but fixed (very low). The deferred
526 * timer might skip some samples if idle/sleeping as needed.
527 */
528 od_dbs_data.min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
529 } else {
530 /* For correct statistics, we need 10 ticks for each measure */
531 od_dbs_data.min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
532 jiffies_to_usecs(10);
533 }
534
535 return cpufreq_register_governor(&cpufreq_gov_ondemand); 586 return cpufreq_register_governor(&cpufreq_gov_ondemand);
536} 587}
537 588
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-22 18:29:16 -0400