/*
* drivers/cpufreq/cpufreq_adaptive.c
*
* Copyright (C) 2001 Russell King
* (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
* Jun Nakajima <jun.nakajima@intel.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/cpufreq.h>
#include <linux/cpu.h>
#include <linux/jiffies.h>
#include <linux/kernel_stat.h>
#include <linux/mutex.h>
#include <linux/hrtimer.h>
#include <linux/tick.h>
#include <linux/ktime.h>
#include <linux/sched.h>
#include <linux/kthread.h>
#include <mach/ppmu.h>
/*
* dbs is used in this file as a shortform for demandbased switching
* It helps to keep variable names smaller, simpler
*/
#define DEF_FREQUENCY_DOWN_DIFFERENTIAL (10)
#define DEF_FREQUENCY_UP_THRESHOLD (80)
#define MICRO_FREQUENCY_DOWN_DIFFERENTIAL (3)
#define MICRO_FREQUENCY_UP_THRESHOLD (95)
#define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000)
#define MIN_FREQUENCY_UP_THRESHOLD (11)
#define MAX_FREQUENCY_UP_THRESHOLD (100)
#define MIN_ONDEMAND_THRESHOLD (4)
/*
* The polling frequency of this governor depends on the capability of
* the processor. Default polling frequency is 1000 times the transition
* latency of the processor. The governor will work on any processor with
* transition latency <= 10mS, using appropriate sampling
* rate.
* For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
* this governor will not work.
* All times here are in uS.
*/
#define MIN_SAMPLING_RATE_RATIO (2)
static unsigned int min_sampling_rate;
#define LATENCY_MULTIPLIER (1000)
#define MIN_LATENCY_MULTIPLIER (100)
#define TRANSITION_LATENCY_LIMIT (10 * 1000 * 1000)
static void (*pm_idle_old)(void);
static void do_dbs_timer(struct work_struct *work);
static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
unsigned int event);
#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ADAPTIVE
static
#endif
struct cpufreq_governor cpufreq_gov_adaptive = {
.name = "adaptive",
.governor = cpufreq_governor_dbs,
.max_transition_latency = TRANSITION_LATENCY_LIMIT,
.owner = THIS_MODULE,
};
/* Sampling types */
enum {DBS_NORMAL_SAMPLE, DBS_SUB_SAMPLE};
struct cpu_dbs_info_s {
cputime64_t prev_cpu_idle;
cputime64_t prev_cpu_iowait;
cputime64_t prev_cpu_wall;
cputime64_t prev_cpu_nice;
struct cpufreq_policy *cur_policy;
struct delayed_work work;
struct cpufreq_frequency_table *freq_table;
unsigned int freq_hi_jiffies;
int cpu;
unsigned int sample_type:1;
bool ondemand;
/*
* percpu mutex that serializes governor limit change with
* do_dbs_timer invocation. We do not want do_dbs_timer to run
* when user is changing the governor or limits.
*/
struct mutex timer_mutex;
};
static DEFINE_PER_CPU(struct cpu_dbs_info_s, od_cpu_dbs_info);
static unsigned int dbs_enable; /* number of CPUs using this policy */
/*
* dbs_mutex protects data in dbs_tuners_ins from concurrent changes on
* different CPUs. It protects dbs_enable in governor start/stop.
*/
static DEFINE_MUTEX(dbs_mutex);
static struct task_struct *up_task;
static struct workqueue_struct *down_wq;
static struct work_struct freq_scale_down_work;
static cpumask_t up_cpumask;
static spinlock_t up_cpumask_lock;
static cpumask_t down_cpumask;
static spinlock_t down_cpumask_lock;
static DEFINE_PER_CPU(cputime64_t, idle_in_idle);
static DEFINE_PER_CPU(cputime64_t, idle_exit_wall);
static struct timer_list cpu_timer;
static unsigned int target_freq;
static DEFINE_MUTEX(short_timer_mutex);
/* Go to max speed when CPU load at or above this value. */
#define DEFAULT_GO_MAXSPEED_LOAD 60
static unsigned long go_maxspeed_load;
#define DEFAULT_KEEP_MINSPEED_LOAD 30
static unsigned long keep_minspeed_load;
#define DEFAULT_STEPUP_LOAD 10
static unsigned long step_up_load;
static struct dbs_tuners {
unsigned int sampling_rate;
unsigned int up_threshold;
unsigned int down_differential;
unsigned int ignore_nice;
unsigned int io_is_busy;
} dbs_tuners_ins = {
.up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
.down_differential = DEF_FREQUENCY_DOWN_DIFFERENTIAL,
.ignore_nice = 0,
};
static inline cputime64_t get_cpu_iowait_time(unsigned int cpu, cputime64_t *wall)
{
u64 iowait_time = get_cpu_iowait_time_us(cpu, wall);
if (iowait_time == -1ULL)
return 0;
return iowait_time;
}
static void adaptive_init_cpu(int cpu)
{
struct cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
dbs_info->freq_table = cpufreq_frequency_get_table(cpu);
}
/************************** sysfs interface ************************/
static ssize_t show_sampling_rate_max(struct kobject *kobj,
struct attribute *attr, char *buf)
{
printk_once(KERN_INFO "CPUFREQ: adaptive sampling_rate_max "
"sysfs file is deprecated - used by: %s\n", current->comm);
return sprintf(buf, "%u\n", -1U);
}
static ssize_t show_sampling_rate_min(struct kobject *kobj,
struct attribute *attr, char *buf)
{
return sprintf(buf, "%u\n", min_sampling_rate);
}
define_one_global_ro(sampling_rate_max);
define_one_global_ro(sampling_rate_min);
/* cpufreq_adaptive Governor Tunables */
#define show_one(file_name, object) \
static ssize_t show_##file_name \
(struct kobject *kobj, struct attribute *attr, char *buf) \
{ \
return sprintf(buf, "%u\n", dbs_tuners_ins.object); \
}
show_one(sampling_rate, sampling_rate);
show_one(io_is_busy, io_is_busy);
show_one(up_threshold, up_threshold);
show_one(ignore_nice_load, ignore_nice);
/*** delete after deprecation time ***/
#define DEPRECATION_MSG(file_name) \
printk_once(KERN_INFO "CPUFREQ: Per core adaptive sysfs " \
"interface is deprecated - " #file_name "\n");
#define show_one_old(file_name) \
static ssize_t show_##file_name##_old \
(struct cpufreq_policy *unused, char *buf) \
{ \
printk_once(KERN_INFO "CPUFREQ: Per core adaptive sysfs " \
"interface is deprecated - " #file_name "\n"); \
return show_##file_name(NULL, NULL, buf); \
}
/*** delete after deprecation time ***/
static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b,
const char *buf, size_t count)
{
unsigned int input;
int ret;
ret = sscanf(buf, "%u", &input);
if (ret != 1)
return -EINVAL;
mutex_lock(&dbs_mutex);
dbs_tuners_ins.sampling_rate = max(input, min_sampling_rate);
mutex_unlock(&dbs_mutex);
return count;
}
static ssize_t store_io_is_busy(struct kobject *a, struct attribute *b,
const char *buf, size_t count)
{
unsigned int input;
int ret;
ret = sscanf(buf, "%u", &input);
if (ret != 1)
return -EINVAL;
mutex_lock(&dbs_mutex);
dbs_tuners_ins.io_is_busy = !!input;
mutex_unlock(&dbs_mutex);
return count;
}
static ssize_t store_up_threshold(struct kobject *a, struct attribute *b,
const char *buf, size_t count)
{
unsigned int input;
int ret;
ret = sscanf(buf, "%u", &input);
if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
input < MIN_FREQUENCY_UP_THRESHOLD) {
return -EINVAL;
}
mutex_lock(&dbs_mutex);
dbs_tuners_ins.up_threshold = input;
mutex_unlock(&dbs_mutex);
return count;
}
static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b,
const char *buf, size_t count)
{
unsigned int input;
int ret;
unsigned int j;
ret = sscanf(buf, "%u", &input);
if (ret != 1)
return -EINVAL;
if (input > 1)
input = 1;
mutex_lock(&dbs_mutex);
if (input == dbs_tuners_ins.ignore_nice) { /* nothing to do */
mutex_unlock(&dbs_mutex);
return count;
}
dbs_tuners_ins.ignore_nice = input;
/* we need to re-evaluate prev_cpu_idle */
for_each_online_cpu(j) {
struct cpu_dbs_info_s *dbs_info;
dbs_info = &per_cpu(od_cpu_dbs_info, j);
dbs_info->prev_cpu_idle = get_cpu_idle_time_us(j,
&dbs_info->prev_cpu_wall);
if (dbs_tuners_ins.ignore_nice)
dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice;
}
mutex_unlock(&dbs_mutex);
return count;
}
define_one_global_rw(sampling_rate);
define_one_global_rw(io_is_busy);
define_one_global_rw(up_threshold);
define_one_global_rw(ignore_nice_load);
static struct attribute *dbs_attributes[] = {
&sampling_rate_max.attr,
&sampling_rate_min.attr,
&sampling_rate.attr,
&up_threshold.attr,
&ignore_nice_load.attr,
&io_is_busy.attr,
NULL
};
static struct attribute_group dbs_attr_group = {
.attrs = dbs_attributes,
.name = "adaptive",
};
/*** delete after deprecation time ***/
#define write_one_old(file_name) \
static ssize_t store_##file_name##_old \
(struct cpufreq_policy *unused, const char *buf, size_t count) \
{ \
printk_once(KERN_INFO "CPUFREQ: Per core adaptive sysfs " \
"interface is deprecated - " #file_name "\n"); \
return store_##file_name(NULL, NULL, buf, count); \
}
static void cpufreq_adaptive_timer(unsigned long data)
{
cputime64_t cur_idle;
cputime64_t cur_wall;
unsigned int delta_idle;
unsigned int delta_time;
int short_load;
unsigned int new_freq;
unsigned long flags;
struct cpu_dbs_info_s *this_dbs_info;
struct cpufreq_policy *policy;
unsigned int j;
unsigned int index;
unsigned int max_load = 0;
this_dbs_info = &per_cpu(od_cpu_dbs_info, 0);
policy = this_dbs_info->cur_policy;
for_each_online_cpu(j) {
cur_idle = get_cpu_idle_time_us(j, &cur_wall);
delta_idle = (unsigned int) cputime64_sub(cur_idle,
per_cpu(idle_in_idle, j));
delta_time = (unsigned int) cputime64_sub(cur_wall,
per_cpu(idle_exit_wall, j));
/*
* If timer ran less than 1ms after short-term sample started, retry.
*/
if (delta_time < 1000)
goto do_nothing;
if (delta_idle > delta_time)
short_load = 0;
else
short_load = 100 * (delta_time - delta_idle) / delta_time;
if (short_load > max_load)
max_load = short_load;
}
if (this_dbs_info->ondemand)
goto do_nothing;
if (max_load >= go_maxspeed_load)
new_freq = policy->max;
else
new_freq = policy->max * max_load / 100;
if ((max_load <= keep_minspeed_load) &&
(policy->cur == policy->min))
new_freq = policy->cur;
if (cpufreq_frequency_table_target(policy, this_dbs_info->freq_table,
new_freq, CPUFREQ_RELATION_L,
&index)) {
goto do_nothing;
}
new_freq = this_dbs_info->freq_table[index].frequency;
target_freq = new_freq;
if (new_freq < this_dbs_info->cur_policy->cur) {
spin_lock_irqsave(&down_cpumask_lock, flags);
cpumask_set_cpu(0, &down_cpumask);
spin_unlock_irqrestore(&down_cpumask_lock, flags);
queue_work(down_wq, &freq_scale_down_work);
} else {
spin_lock_irqsave(&up_cpumask_lock, flags);
cpumask_set_cpu(0, &up_cpumask);
spin_unlock_irqrestore(&up_cpumask_lock, flags);
wake_up_process(up_task);
}
return;
do_nothing:
for_each_online_cpu(j) {
per_cpu(idle_in_idle, j) =
get_cpu_idle_time_us(j,
&per_cpu(idle_exit_wall, j));
}
mod_timer(&cpu_timer, jiffies + 2);
schedule_delayed_work_on(0, &this_dbs_info->work, 10);
if (mutex_is_locked(&short_timer_mutex))
mutex_unlock(&short_timer_mutex);
return;
}
/*** delete after deprecation time ***/
/************************** sysfs end ************************/
static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq)
{
#ifndef CONFIG_ARCH_EXYNOS4
if (p->cur == p->max)
return;
#endif
__cpufreq_driver_target(p, freq, CPUFREQ_RELATION_H);
}
static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
{
unsigned int max_load_freq;
struct cpufreq_policy *policy;
unsigned int j;
unsigned int index, new_freq;
unsigned int longterm_load = 0;
policy = this_dbs_info->cur_policy;
/*
* Every sampling_rate, we check, if current idle time is less
* than 20% (default), then we try to increase frequency
* Every sampling_rate, we look for a the lowest
* frequency which can sustain the load while keeping idle time over
* 30%. If such a frequency exist, we try to decrease to this frequency.
*
* Any frequency increase takes it to the maximum frequency.
* Frequency reduction happens at minimum steps of
* 5% (default) of current frequency
*/
/* Get Absolute Load - in terms of freq */
max_load_freq = 0;
for_each_cpu(j, policy->cpus) {
struct cpu_dbs_info_s *j_dbs_info;
cputime64_t cur_wall_time, cur_idle_time, cur_iowait_time;
unsigned int idle_time, wall_time, iowait_time;
unsigned int load, load_freq;
int freq_avg;
j_dbs_info = &per_cpu(od_cpu_dbs_info, j);
cur_idle_time = get_cpu_idle_time_us(j, &cur_wall_time);
cur_iowait_time = get_cpu_iowait_time(j, &cur_wall_time);
wall_time = (unsigned int) cputime64_sub(cur_wall_time,
j_dbs_info->prev_cpu_wall);
j_dbs_info->prev_cpu_wall = cur_wall_time;
idle_time = (unsigned int) cputime64_sub(cur_idle_time,
j_dbs_info->prev_cpu_idle);
j_dbs_info->prev_cpu_idle = cur_idle_time;
iowait_time = (unsigned int) cputime64_sub(cur_iowait_time,
j_dbs_info->prev_cpu_iowait);
j_dbs_info->prev_cpu_iowait = cur_iowait_time;
if (dbs_tuners_ins.ignore_nice) {
cputime64_t cur_nice;
unsigned long cur_nice_jiffies;
cur_nice = cputime64_sub(kstat_cpu(j).cpustat.nice,
j_dbs_info->prev_cpu_nice);
/*
* Assumption: nice time between sampling periods will
* be less than 2^32 jiffies for 32 bit sys
*/
cur_nice_jiffies = (unsigned long)
cputime64_to_jiffies64(cur_nice);
j_dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice;
idle_time += jiffies_to_usecs(cur_nice_jiffies);
}
/*
* For the purpose of adaptive, waiting for disk IO is an
* indication that you're performance critical, and not that
* the system is actually idle. So subtract the iowait time
* from the cpu idle time.
*/
if (dbs_tuners_ins.io_is_busy && idle_time >= iowait_time)
idle_time -= iowait_time;
if (unlikely(!wall_time || wall_time < idle_time))
continue;
load = 100 * (wall_time - idle_time) / wall_time;
if (load > longterm_load)
longterm_load = load;
freq_avg = __cpufreq_driver_getavg(policy, j);
if (freq_avg <= 0)
freq_avg = policy->cur;
load_freq = load * freq_avg;
if (load_freq > max_load_freq)
max_load_freq = load_freq;
}
if (longterm_load >= MIN_ONDEMAND_THRESHOLD)
this_dbs_info->ondemand = true;
else
this_dbs_info->ondemand = false;
/* Check for frequency increase */
if (max_load_freq > (dbs_tuners_ins.up_threshold * policy->cur)) {
cpufreq_frequency_table_target(policy,
this_dbs_info->freq_table,
(policy->cur + step_up_load),
CPUFREQ_RELATION_L, &index);
new_freq = this_dbs_info->freq_table[index].frequency;
dbs_freq_increase(policy, new_freq);
return;
}
/* Check for frequency decrease */
/* if we cannot reduce the frequency anymore, break out early */
#ifndef CONFIG_ARCH_EXYNOS4
if (policy->cur == policy->min)
return;
#endif
/*
* The optimal frequency is the frequency that is the lowest that
* can support the current CPU usage without triggering the up
* policy. To be safe, we focus 10 points under the threshold.
*/
if (max_load_freq <
(dbs_tuners_ins.up_threshold - dbs_tuners_ins.down_differential) *
policy->cur) {
unsigned int freq_next;
freq_next = max_load_freq /
(dbs_tuners_ins.up_threshold -
dbs_tuners_ins.down_differential);
if (freq_next < policy->min)
freq_next = policy->min;
__cpufreq_driver_target(policy, freq_next,
CPUFREQ_RELATION_L);
}
}
static void do_dbs_timer(struct work_struct *work)
{
struct cpu_dbs_info_s *dbs_info =
container_of(work, struct cpu_dbs_info_s, work.work);
unsigned int cpu = dbs_info->cpu;
int delay;
mutex_lock(&dbs_info->timer_mutex);
/* Common NORMAL_SAMPLE setup */
dbs_info->sample_type = DBS_NORMAL_SAMPLE;
dbs_check_cpu(dbs_info);
/* We want all CPUs to do sampling nearly on
* same jiffy
*/
delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
schedule_delayed_work_on(cpu, &dbs_info->work, delay);
mutex_unlock(&dbs_info->timer_mutex);
}
static inline void dbs_timer_init(struct cpu_dbs_info_s *dbs_info)
{
/* We want all CPUs to do sampling nearly on same jiffy */
int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
dbs_info->sample_type = DBS_NORMAL_SAMPLE;
INIT_DELAYED_WORK_DEFERRABLE(&dbs_info->work, do_dbs_timer);
schedule_delayed_work_on(dbs_info->cpu, &dbs_info->work, delay);
}
static inline void dbs_timer_exit(struct cpu_dbs_info_s *dbs_info)
{
cancel_delayed_work_sync(&dbs_info->work);
}
/*
* Not all CPUs want IO time to be accounted as busy; this dependson how
* efficient idling at a higher frequency/voltage is.
* Pavel Machek says this is not so for various generations of AMD and old
* Intel systems.
* Mike Chan (androidlcom) calis this is also not true for ARM.
* Because of this, whitelist specific known (series) of CPUs by default, and
* leave all others up to the user.
*/
static int should_io_be_busy(void)
{
#if defined(CONFIG_X86)
/*
* For Intel, Core 2 (model 15) andl later have an efficient idle.
*/
if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
boot_cpu_data.x86 == 6 &&
boot_cpu_data.x86_model >= 15)
return 1;
#endif
return 0;
}
static void cpufreq_adaptive_idle(void)
{
int i;
struct cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, 0);
struct cpufreq_policy *policy;
policy = dbs_info->cur_policy;
pm_idle_old();
if ((policy->cur == policy->min) ||
(policy->cur == policy->max)) {
if (timer_pending(&cpu_timer))
return;
if (mutex_trylock(&short_timer_mutex)) {
for_each_online_cpu(i) {
per_cpu(idle_in_idle, i) =
get_cpu_idle_time_us(i,
&per_cpu(idle_exit_wall, i));
}
mod_timer(&cpu_timer, jiffies + 2);
cancel_delayed_work(&dbs_info->work);
}
} else {
if (timer_pending(&cpu_timer))
del_timer(&cpu_timer);
}
}
static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
unsigned int event)
{
unsigned int cpu = policy->cpu;
struct cpu_dbs_info_s *this_dbs_info;
unsigned int j;
int rc;
this_dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
switch (event) {
case CPUFREQ_GOV_START:
if ((!cpu_online(cpu)) || (!policy->cur))
return -EINVAL;
mutex_lock(&dbs_mutex);
rc = sysfs_create_group(&policy->kobj, &dbs_attr_group);
if (rc) {
mutex_unlock(&dbs_mutex);
return rc;
}
dbs_enable++;
for_each_cpu(j, policy->cpus) {
struct cpu_dbs_info_s *j_dbs_info;
j_dbs_info = &per_cpu(od_cpu_dbs_info, j);
j_dbs_info->cur_policy = policy;
j_dbs_info->prev_cpu_idle = get_cpu_idle_time_us(j,
&j_dbs_info->prev_cpu_wall);
if (dbs_tuners_ins.ignore_nice) {
j_dbs_info->prev_cpu_nice =
kstat_cpu(j).cpustat.nice;
}
}
this_dbs_info->cpu = cpu;
adaptive_init_cpu(cpu);
/*
* Start the timerschedule work, when this governor
* is used for first time
*/
if (dbs_enable == 1) {
unsigned int latency;
rc = sysfs_create_group(cpufreq_global_kobject,
&dbs_attr_group);
if (rc) {
mutex_unlock(&dbs_mutex);
return rc;
}
/* policy latency is in nS. Convert it to uS first */
latency = policy->cpuinfo.transition_latency / 1000;
if (latency == 0)
latency = 1;
/* Bring kernel and HW constraints together */
min_sampling_rate = max(min_sampling_rate,
MIN_LATENCY_MULTIPLIER * latency);
dbs_tuners_ins.sampling_rate =
max(min_sampling_rate,
latency * LATENCY_MULTIPLIER);
dbs_tuners_ins.io_is_busy = should_io_be_busy();
}
mutex_unlock(&dbs_mutex);
mutex_init(&this_dbs_info->timer_mutex);
dbs_timer_init(this_dbs_info);
pm_idle_old = pm_idle;
pm_idle = cpufreq_adaptive_idle;
break;
case CPUFREQ_GOV_STOP:
dbs_timer_exit(this_dbs_info);
mutex_lock(&dbs_mutex);
sysfs_remove_group(&policy->kobj, &dbs_attr_group);
mutex_destroy(&this_dbs_info->timer_mutex);
dbs_enable--;
mutex_unlock(&dbs_mutex);
if (!dbs_enable)
sysfs_remove_group(cpufreq_global_kobject,
&dbs_attr_group);
pm_idle = pm_idle_old;
break;
case CPUFREQ_GOV_LIMITS:
mutex_lock(&this_dbs_info->timer_mutex);
if (policy->max < this_dbs_info->cur_policy->cur)
__cpufreq_driver_target(this_dbs_info->cur_policy,
policy->max, CPUFREQ_RELATION_H);
else if (policy->min > this_dbs_info->cur_policy->cur)
__cpufreq_driver_target(this_dbs_info->cur_policy,
policy->min, CPUFREQ_RELATION_L);
mutex_unlock(&this_dbs_info->timer_mutex);
break;
}
return 0;
}
static inline void cpufreq_adaptive_update_time(void)
{
struct cpu_dbs_info_s *this_dbs_info;
struct cpufreq_policy *policy;
int j;
this_dbs_info = &per_cpu(od_cpu_dbs_info, 0);
policy = this_dbs_info->cur_policy;
for_each_cpu(j, policy->cpus) {
struct cpu_dbs_info_s *j_dbs_info;
cputime64_t cur_wall_time, cur_idle_time, cur_iowait_time;
j_dbs_info = &per_cpu(od_cpu_dbs_info, j);
cur_idle_time = get_cpu_idle_time_us(j, &cur_wall_time);
cur_iowait_time = get_cpu_iowait_time(j, &cur_wall_time);
j_dbs_info->prev_cpu_wall = cur_wall_time;
j_dbs_info->prev_cpu_idle = cur_idle_time;
j_dbs_info->prev_cpu_iowait = cur_iowait_time;
if (dbs_tuners_ins.ignore_nice)
j_dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice;
}
}
static int cpufreq_adaptive_up_task(void *data)
{
unsigned long flags;
struct cpu_dbs_info_s *this_dbs_info;
struct cpufreq_policy *policy;
int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
this_dbs_info = &per_cpu(od_cpu_dbs_info, 0);
policy = this_dbs_info->cur_policy;
while (1) {
set_current_state(TASK_INTERRUPTIBLE);
spin_lock_irqsave(&up_cpumask_lock, flags);
if (cpumask_empty(&up_cpumask)) {
spin_unlock_irqrestore(&up_cpumask_lock, flags);
schedule();
if (kthread_should_stop())
break;
spin_lock_irqsave(&up_cpumask_lock, flags);
}
set_current_state(TASK_RUNNING);
cpumask_clear(&up_cpumask);
spin_unlock_irqrestore(&up_cpumask_lock, flags);
__cpufreq_driver_target(this_dbs_info->cur_policy,
target_freq,
CPUFREQ_RELATION_H);
if (policy->cur != policy->max) {
mutex_lock(&this_dbs_info->timer_mutex);
schedule_delayed_work_on(0, &this_dbs_info->work, delay);
mutex_unlock(&this_dbs_info->timer_mutex);
cpufreq_adaptive_update_time();
}
if (mutex_is_locked(&short_timer_mutex))
mutex_unlock(&short_timer_mutex);
}
return 0;
}
static void cpufreq_adaptive_freq_down(struct work_struct *work)
{
unsigned long flags;
struct cpu_dbs_info_s *this_dbs_info;
struct cpufreq_policy *policy;
int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
spin_lock_irqsave(&down_cpumask_lock, flags);
cpumask_clear(&down_cpumask);
spin_unlock_irqrestore(&down_cpumask_lock, flags);
this_dbs_info = &per_cpu(od_cpu_dbs_info, 0);
policy = this_dbs_info->cur_policy;
__cpufreq_driver_target(this_dbs_info->cur_policy,
target_freq,
CPUFREQ_RELATION_H);
if (policy->cur != policy->min) {
mutex_lock(&this_dbs_info->timer_mutex);
schedule_delayed_work_on(0, &this_dbs_info->work, delay);
mutex_unlock(&this_dbs_info->timer_mutex);
cpufreq_adaptive_update_time();
}
if (mutex_is_locked(&short_timer_mutex))
mutex_unlock(&short_timer_mutex);
}
static int __init cpufreq_gov_dbs_init(void)
{
cputime64_t wall;
u64 idle_time;
int cpu = get_cpu();
struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
go_maxspeed_load = DEFAULT_GO_MAXSPEED_LOAD;
keep_minspeed_load = DEFAULT_KEEP_MINSPEED_LOAD;
step_up_load = DEFAULT_STEPUP_LOAD;
idle_time = get_cpu_idle_time_us(cpu, &wall);
put_cpu();
if (idle_time != -1ULL) {
/* Idle micro accounting is supported. Use finer thresholds */
dbs_tuners_ins.up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
dbs_tuners_ins.down_differential =
MICRO_FREQUENCY_DOWN_DIFFERENTIAL;
/*
* In no_hz/micro accounting case we set the minimum frequency
* not depending on HZ, but fixed (very low). The deferred
* timer might skip some samples if idle/sleeping as needed.
*/
min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
} else {
/* For correct statistics, we need 10 ticks for each measure */
min_sampling_rate =
MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10);
}
init_timer(&cpu_timer);
cpu_timer.function = cpufreq_adaptive_timer;
up_task = kthread_create(cpufreq_adaptive_up_task, NULL,
"kadaptiveup");
if (IS_ERR(up_task))
return PTR_ERR(up_task);
sched_setscheduler_nocheck(up_task, SCHED_FIFO, ¶m);
get_task_struct(up_task);
/* No rescuer thread, bind to CPU queuing the work for possibly
warm cache (probably doesn't matter much). */
down_wq = alloc_workqueue("kadaptive_down", 0, 1);
if (!down_wq)
goto err_freeuptask;
INIT_WORK(&freq_scale_down_work, cpufreq_adaptive_freq_down);
return cpufreq_register_governor(&cpufreq_gov_adaptive);
err_freeuptask:
put_task_struct(up_task);
return -ENOMEM;
}
static void __exit cpufreq_gov_dbs_exit(void)
{
cpufreq_unregister_governor(&cpufreq_gov_adaptive);
}
MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
MODULE_DESCRIPTION("'cpufreq_adaptive' - A dynamic cpufreq governor for "
"Low Latency Frequency Transition capable processors");
MODULE_LICENSE("GPL");
#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ADAPTIVE
fs_initcall(cpufreq_gov_dbs_init);
#else
module_init(cpufreq_gov_dbs_init);
#endif
module_exit(cpufreq_gov_dbs_exit);
