aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/cpufreq
diff options
context:
space:
mode:
authorRafael J. Wysocki <rafael.j.wysocki@intel.com>2016-03-10 14:46:03 -0500
committerRafael J. Wysocki <rafael.j.wysocki@intel.com>2016-03-10 14:46:03 -0500
commita5acbfbd70bc6a0cd4c8ca8f4f5e52ac3ee7dca4 (patch)
tree5e4560aaaaf2dec73a25bf448e0dd73ded4f9117 /drivers/cpufreq
parentedd4a893e097d744e8069acf585f8b02dbbc9134 (diff)
parentadaf9fcd136970e480d7ca834c0cf25ce922ea74 (diff)
Merge branch 'pm-cpufreq-governor' into pm-cpufreq
Diffstat (limited to 'drivers/cpufreq')
-rw-r--r--drivers/cpufreq/Kconfig1
-rw-r--r--drivers/cpufreq/amd_freq_sensitivity.c8
-rw-r--r--drivers/cpufreq/cpufreq.c165
-rw-r--r--drivers/cpufreq/cpufreq_conservative.c276
-rw-r--r--drivers/cpufreq/cpufreq_governor.c766
-rw-r--r--drivers/cpufreq/cpufreq_governor.h261
-rw-r--r--drivers/cpufreq/cpufreq_ondemand.c435
-rw-r--r--drivers/cpufreq/cpufreq_ondemand.h30
-rw-r--r--drivers/cpufreq/intel_pstate.c103
9 files changed, 903 insertions, 1142 deletions
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index 659879a56dba..aa403aa2b927 100644
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -19,6 +19,7 @@ config CPU_FREQ
19if CPU_FREQ 19if CPU_FREQ
20 20
21config CPU_FREQ_GOV_COMMON 21config CPU_FREQ_GOV_COMMON
22 select IRQ_WORK
22 bool 23 bool
23 24
24config CPU_FREQ_BOOST_SW 25config CPU_FREQ_BOOST_SW
diff --git a/drivers/cpufreq/amd_freq_sensitivity.c b/drivers/cpufreq/amd_freq_sensitivity.c
index f6b79ab0070b..404360cad25c 100644
--- a/drivers/cpufreq/amd_freq_sensitivity.c
+++ b/drivers/cpufreq/amd_freq_sensitivity.c
@@ -21,7 +21,7 @@
21#include <asm/msr.h> 21#include <asm/msr.h>
22#include <asm/cpufeature.h> 22#include <asm/cpufeature.h>
23 23
24#include "cpufreq_governor.h" 24#include "cpufreq_ondemand.h"
25 25
26#define MSR_AMD64_FREQ_SENSITIVITY_ACTUAL 0xc0010080 26#define MSR_AMD64_FREQ_SENSITIVITY_ACTUAL 0xc0010080
27#define MSR_AMD64_FREQ_SENSITIVITY_REFERENCE 0xc0010081 27#define MSR_AMD64_FREQ_SENSITIVITY_REFERENCE 0xc0010081
@@ -45,10 +45,10 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy,
45 long d_actual, d_reference; 45 long d_actual, d_reference;
46 struct msr actual, reference; 46 struct msr actual, reference;
47 struct cpu_data_t *data = &per_cpu(cpu_data, policy->cpu); 47 struct cpu_data_t *data = &per_cpu(cpu_data, policy->cpu);
48 struct dbs_data *od_data = policy->governor_data; 48 struct policy_dbs_info *policy_dbs = policy->governor_data;
49 struct dbs_data *od_data = policy_dbs->dbs_data;
49 struct od_dbs_tuners *od_tuners = od_data->tuners; 50 struct od_dbs_tuners *od_tuners = od_data->tuners;
50 struct od_cpu_dbs_info_s *od_info = 51 struct od_policy_dbs_info *od_info = to_dbs_info(policy_dbs);
51 od_data->cdata->get_cpu_dbs_info_s(policy->cpu);
52 52
53 if (!od_info->freq_table) 53 if (!od_info->freq_table)
54 return freq_next; 54 return freq_next;
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index abca44c2e4e3..4c7825856eab 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -64,7 +64,6 @@ static LIST_HEAD(cpufreq_governor_list);
64static struct cpufreq_driver *cpufreq_driver; 64static struct cpufreq_driver *cpufreq_driver;
65static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); 65static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
66static DEFINE_RWLOCK(cpufreq_driver_lock); 66static DEFINE_RWLOCK(cpufreq_driver_lock);
67DEFINE_MUTEX(cpufreq_governor_lock);
68 67
69/* Flag to suspend/resume CPUFreq governors */ 68/* Flag to suspend/resume CPUFreq governors */
70static bool cpufreq_suspended; 69static bool cpufreq_suspended;
@@ -75,10 +74,8 @@ static inline bool has_target(void)
75} 74}
76 75
77/* internal prototypes */ 76/* internal prototypes */
78static int __cpufreq_governor(struct cpufreq_policy *policy, 77static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
79 unsigned int event);
80static unsigned int __cpufreq_get(struct cpufreq_policy *policy); 78static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
81static void handle_update(struct work_struct *work);
82 79
83/** 80/**
84 * Two notifier lists: the "policy" list is involved in the 81 * Two notifier lists: the "policy" list is involved in the
@@ -955,30 +952,38 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cp
955 if (cpumask_test_cpu(cpu, policy->cpus)) 952 if (cpumask_test_cpu(cpu, policy->cpus))
956 return 0; 953 return 0;
957 954
955 down_write(&policy->rwsem);
958 if (has_target()) { 956 if (has_target()) {
959 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); 957 ret = cpufreq_governor(policy, CPUFREQ_GOV_STOP);
960 if (ret) { 958 if (ret) {
961 pr_err("%s: Failed to stop governor\n", __func__); 959 pr_err("%s: Failed to stop governor\n", __func__);
962 return ret; 960 goto unlock;
963 } 961 }
964 } 962 }
965 963
966 down_write(&policy->rwsem);
967 cpumask_set_cpu(cpu, policy->cpus); 964 cpumask_set_cpu(cpu, policy->cpus);
968 up_write(&policy->rwsem);
969 965
970 if (has_target()) { 966 if (has_target()) {
971 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START); 967 ret = cpufreq_governor(policy, CPUFREQ_GOV_START);
972 if (!ret) 968 if (!ret)
973 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 969 ret = cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
974 970
975 if (ret) { 971 if (ret)
976 pr_err("%s: Failed to start governor\n", __func__); 972 pr_err("%s: Failed to start governor\n", __func__);
977 return ret;
978 }
979 } 973 }
980 974
981 return 0; 975unlock:
976 up_write(&policy->rwsem);
977 return ret;
978}
979
980static void handle_update(struct work_struct *work)
981{
982 struct cpufreq_policy *policy =
983 container_of(work, struct cpufreq_policy, update);
984 unsigned int cpu = policy->cpu;
985 pr_debug("handle_update for cpu %u called\n", cpu);
986 cpufreq_update_policy(cpu);
982} 987}
983 988
984static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) 989static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
@@ -1267,9 +1272,10 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1267 return ret; 1272 return ret;
1268} 1273}
1269 1274
1270static void cpufreq_offline_prepare(unsigned int cpu) 1275static void cpufreq_offline(unsigned int cpu)
1271{ 1276{
1272 struct cpufreq_policy *policy; 1277 struct cpufreq_policy *policy;
1278 int ret;
1273 1279
1274 pr_debug("%s: unregistering CPU %u\n", __func__, cpu); 1280 pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1275 1281
@@ -1279,13 +1285,13 @@ static void cpufreq_offline_prepare(unsigned int cpu)
1279 return; 1285 return;
1280 } 1286 }
1281 1287
1288 down_write(&policy->rwsem);
1282 if (has_target()) { 1289 if (has_target()) {
1283 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); 1290 ret = cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1284 if (ret) 1291 if (ret)
1285 pr_err("%s: Failed to stop governor\n", __func__); 1292 pr_err("%s: Failed to stop governor\n", __func__);
1286 } 1293 }
1287 1294
1288 down_write(&policy->rwsem);
1289 cpumask_clear_cpu(cpu, policy->cpus); 1295 cpumask_clear_cpu(cpu, policy->cpus);
1290 1296
1291 if (policy_is_inactive(policy)) { 1297 if (policy_is_inactive(policy)) {
@@ -1298,39 +1304,27 @@ static void cpufreq_offline_prepare(unsigned int cpu)
1298 /* Nominate new CPU */ 1304 /* Nominate new CPU */
1299 policy->cpu = cpumask_any(policy->cpus); 1305 policy->cpu = cpumask_any(policy->cpus);
1300 } 1306 }
1301 up_write(&policy->rwsem);
1302 1307
1303 /* Start governor again for active policy */ 1308 /* Start governor again for active policy */
1304 if (!policy_is_inactive(policy)) { 1309 if (!policy_is_inactive(policy)) {
1305 if (has_target()) { 1310 if (has_target()) {
1306 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_START); 1311 ret = cpufreq_governor(policy, CPUFREQ_GOV_START);
1307 if (!ret) 1312 if (!ret)
1308 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 1313 ret = cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1309 1314
1310 if (ret) 1315 if (ret)
1311 pr_err("%s: Failed to start governor\n", __func__); 1316 pr_err("%s: Failed to start governor\n", __func__);
1312 } 1317 }
1313 } else if (cpufreq_driver->stop_cpu) {
1314 cpufreq_driver->stop_cpu(policy);
1315 }
1316}
1317 1318
1318static void cpufreq_offline_finish(unsigned int cpu) 1319 goto unlock;
1319{
1320 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1321
1322 if (!policy) {
1323 pr_debug("%s: No cpu_data found\n", __func__);
1324 return;
1325 } 1320 }
1326 1321
1327 /* Only proceed for inactive policies */ 1322 if (cpufreq_driver->stop_cpu)
1328 if (!policy_is_inactive(policy)) 1323 cpufreq_driver->stop_cpu(policy);
1329 return;
1330 1324
1331 /* If cpu is last user of policy, free policy */ 1325 /* If cpu is last user of policy, free policy */
1332 if (has_target()) { 1326 if (has_target()) {
1333 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); 1327 ret = cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1334 if (ret) 1328 if (ret)
1335 pr_err("%s: Failed to exit governor\n", __func__); 1329 pr_err("%s: Failed to exit governor\n", __func__);
1336 } 1330 }
@@ -1344,6 +1338,9 @@ static void cpufreq_offline_finish(unsigned int cpu)
1344 cpufreq_driver->exit(policy); 1338 cpufreq_driver->exit(policy);
1345 policy->freq_table = NULL; 1339 policy->freq_table = NULL;
1346 } 1340 }
1341
1342unlock:
1343 up_write(&policy->rwsem);
1347} 1344}
1348 1345
1349/** 1346/**
@@ -1359,10 +1356,8 @@ static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1359 if (!policy) 1356 if (!policy)
1360 return; 1357 return;
1361 1358
1362 if (cpu_online(cpu)) { 1359 if (cpu_online(cpu))
1363 cpufreq_offline_prepare(cpu); 1360 cpufreq_offline(cpu);
1364 cpufreq_offline_finish(cpu);
1365 }
1366 1361
1367 cpumask_clear_cpu(cpu, policy->real_cpus); 1362 cpumask_clear_cpu(cpu, policy->real_cpus);
1368 remove_cpu_dev_symlink(policy, cpu); 1363 remove_cpu_dev_symlink(policy, cpu);
@@ -1371,15 +1366,6 @@ static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1371 cpufreq_policy_free(policy, true); 1366 cpufreq_policy_free(policy, true);
1372} 1367}
1373 1368
1374static void handle_update(struct work_struct *work)
1375{
1376 struct cpufreq_policy *policy =
1377 container_of(work, struct cpufreq_policy, update);
1378 unsigned int cpu = policy->cpu;
1379 pr_debug("handle_update for cpu %u called\n", cpu);
1380 cpufreq_update_policy(cpu);
1381}
1382
1383/** 1369/**
1384 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're 1370 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1385 * in deep trouble. 1371 * in deep trouble.
@@ -1542,6 +1528,7 @@ EXPORT_SYMBOL(cpufreq_generic_suspend);
1542void cpufreq_suspend(void) 1528void cpufreq_suspend(void)
1543{ 1529{
1544 struct cpufreq_policy *policy; 1530 struct cpufreq_policy *policy;
1531 int ret;
1545 1532
1546 if (!cpufreq_driver) 1533 if (!cpufreq_driver)
1547 return; 1534 return;
@@ -1552,7 +1539,11 @@ void cpufreq_suspend(void)
1552 pr_debug("%s: Suspending Governors\n", __func__); 1539 pr_debug("%s: Suspending Governors\n", __func__);
1553 1540
1554 for_each_active_policy(policy) { 1541 for_each_active_policy(policy) {
1555 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP)) 1542 down_write(&policy->rwsem);
1543 ret = cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1544 up_write(&policy->rwsem);
1545
1546 if (ret)
1556 pr_err("%s: Failed to stop governor for policy: %p\n", 1547 pr_err("%s: Failed to stop governor for policy: %p\n",
1557 __func__, policy); 1548 __func__, policy);
1558 else if (cpufreq_driver->suspend 1549 else if (cpufreq_driver->suspend
@@ -1574,6 +1565,7 @@ suspend:
1574void cpufreq_resume(void) 1565void cpufreq_resume(void)
1575{ 1566{
1576 struct cpufreq_policy *policy; 1567 struct cpufreq_policy *policy;
1568 int ret;
1577 1569
1578 if (!cpufreq_driver) 1570 if (!cpufreq_driver)
1579 return; 1571 return;
@@ -1586,13 +1578,20 @@ void cpufreq_resume(void)
1586 pr_debug("%s: Resuming Governors\n", __func__); 1578 pr_debug("%s: Resuming Governors\n", __func__);
1587 1579
1588 for_each_active_policy(policy) { 1580 for_each_active_policy(policy) {
1589 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) 1581 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
1590 pr_err("%s: Failed to resume driver: %p\n", __func__, 1582 pr_err("%s: Failed to resume driver: %p\n", __func__,
1591 policy); 1583 policy);
1592 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START) 1584 } else {
1593 || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS)) 1585 down_write(&policy->rwsem);
1594 pr_err("%s: Failed to start governor for policy: %p\n", 1586 ret = cpufreq_governor(policy, CPUFREQ_GOV_START);
1595 __func__, policy); 1587 if (!ret)
1588 cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1589 up_write(&policy->rwsem);
1590
1591 if (ret)
1592 pr_err("%s: Failed to start governor for policy: %p\n",
1593 __func__, policy);
1594 }
1596 } 1595 }
1597 1596
1598 /* 1597 /*
@@ -1878,8 +1877,7 @@ __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
1878 return NULL; 1877 return NULL;
1879} 1878}
1880 1879
1881static int __cpufreq_governor(struct cpufreq_policy *policy, 1880static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event)
1882 unsigned int event)
1883{ 1881{
1884 int ret; 1882 int ret;
1885 1883
@@ -1913,21 +1911,6 @@ static int __cpufreq_governor(struct cpufreq_policy *policy,
1913 1911
1914 pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event); 1912 pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event);
1915 1913
1916 mutex_lock(&cpufreq_governor_lock);
1917 if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1918 || (!policy->governor_enabled
1919 && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1920 mutex_unlock(&cpufreq_governor_lock);
1921 return -EBUSY;
1922 }
1923
1924 if (event == CPUFREQ_GOV_STOP)
1925 policy->governor_enabled = false;
1926 else if (event == CPUFREQ_GOV_START)
1927 policy->governor_enabled = true;
1928
1929 mutex_unlock(&cpufreq_governor_lock);
1930
1931 ret = policy->governor->governor(policy, event); 1914 ret = policy->governor->governor(policy, event);
1932 1915
1933 if (!ret) { 1916 if (!ret) {
@@ -1935,14 +1918,6 @@ static int __cpufreq_governor(struct cpufreq_policy *policy,
1935 policy->governor->initialized++; 1918 policy->governor->initialized++;
1936 else if (event == CPUFREQ_GOV_POLICY_EXIT) 1919 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1937 policy->governor->initialized--; 1920 policy->governor->initialized--;
1938 } else {
1939 /* Restore original values */
1940 mutex_lock(&cpufreq_governor_lock);
1941 if (event == CPUFREQ_GOV_STOP)
1942 policy->governor_enabled = true;
1943 else if (event == CPUFREQ_GOV_START)
1944 policy->governor_enabled = false;
1945 mutex_unlock(&cpufreq_governor_lock);
1946 } 1921 }
1947 1922
1948 if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) || 1923 if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
@@ -2097,7 +2072,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2097 old_gov = policy->governor; 2072 old_gov = policy->governor;
2098 /* end old governor */ 2073 /* end old governor */
2099 if (old_gov) { 2074 if (old_gov) {
2100 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); 2075 ret = cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2101 if (ret) { 2076 if (ret) {
2102 /* This can happen due to race with other operations */ 2077 /* This can happen due to race with other operations */
2103 pr_debug("%s: Failed to Stop Governor: %s (%d)\n", 2078 pr_debug("%s: Failed to Stop Governor: %s (%d)\n",
@@ -2105,10 +2080,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2105 return ret; 2080 return ret;
2106 } 2081 }
2107 2082
2108 up_write(&policy->rwsem); 2083 ret = cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2109 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2110 down_write(&policy->rwsem);
2111
2112 if (ret) { 2084 if (ret) {
2113 pr_err("%s: Failed to Exit Governor: %s (%d)\n", 2085 pr_err("%s: Failed to Exit Governor: %s (%d)\n",
2114 __func__, old_gov->name, ret); 2086 __func__, old_gov->name, ret);
@@ -2118,32 +2090,30 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2118 2090
2119 /* start new governor */ 2091 /* start new governor */
2120 policy->governor = new_policy->governor; 2092 policy->governor = new_policy->governor;
2121 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT); 2093 ret = cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2122 if (!ret) { 2094 if (!ret) {
2123 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START); 2095 ret = cpufreq_governor(policy, CPUFREQ_GOV_START);
2124 if (!ret) 2096 if (!ret)
2125 goto out; 2097 goto out;
2126 2098
2127 up_write(&policy->rwsem); 2099 cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2128 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2129 down_write(&policy->rwsem);
2130 } 2100 }
2131 2101
2132 /* new governor failed, so re-start old one */ 2102 /* new governor failed, so re-start old one */
2133 pr_debug("starting governor %s failed\n", policy->governor->name); 2103 pr_debug("starting governor %s failed\n", policy->governor->name);
2134 if (old_gov) { 2104 if (old_gov) {
2135 policy->governor = old_gov; 2105 policy->governor = old_gov;
2136 if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) 2106 if (cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT))
2137 policy->governor = NULL; 2107 policy->governor = NULL;
2138 else 2108 else
2139 __cpufreq_governor(policy, CPUFREQ_GOV_START); 2109 cpufreq_governor(policy, CPUFREQ_GOV_START);
2140 } 2110 }
2141 2111
2142 return ret; 2112 return ret;
2143 2113
2144 out: 2114 out:
2145 pr_debug("governor: change or update limits\n"); 2115 pr_debug("governor: change or update limits\n");
2146 return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 2116 return cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2147} 2117}
2148 2118
2149/** 2119/**
@@ -2210,11 +2180,7 @@ static int cpufreq_cpu_callback(struct notifier_block *nfb,
2210 break; 2180 break;
2211 2181
2212 case CPU_DOWN_PREPARE: 2182 case CPU_DOWN_PREPARE:
2213 cpufreq_offline_prepare(cpu); 2183 cpufreq_offline(cpu);
2214 break;
2215
2216 case CPU_POST_DEAD:
2217 cpufreq_offline_finish(cpu);
2218 break; 2184 break;
2219 2185
2220 case CPU_DOWN_FAILED: 2186 case CPU_DOWN_FAILED:
@@ -2247,8 +2213,11 @@ static int cpufreq_boost_set_sw(int state)
2247 __func__); 2213 __func__);
2248 break; 2214 break;
2249 } 2215 }
2216
2217 down_write(&policy->rwsem);
2250 policy->user_policy.max = policy->max; 2218 policy->user_policy.max = policy->max;
2251 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 2219 cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2220 up_write(&policy->rwsem);
2252 } 2221 }
2253 } 2222 }
2254 2223
diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c
index 8504a70a4785..bf4913f6453b 100644
--- a/drivers/cpufreq/cpufreq_conservative.c
+++ b/drivers/cpufreq/cpufreq_conservative.c
@@ -14,6 +14,22 @@
14#include <linux/slab.h> 14#include <linux/slab.h>
15#include "cpufreq_governor.h" 15#include "cpufreq_governor.h"
16 16
17struct cs_policy_dbs_info {
18 struct policy_dbs_info policy_dbs;
19 unsigned int down_skip;
20 unsigned int requested_freq;
21};
22
23static inline struct cs_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs)
24{
25 return container_of(policy_dbs, struct cs_policy_dbs_info, policy_dbs);
26}
27
28struct cs_dbs_tuners {
29 unsigned int down_threshold;
30 unsigned int freq_step;
31};
32
17/* Conservative governor macros */ 33/* Conservative governor macros */
18#define DEF_FREQUENCY_UP_THRESHOLD (80) 34#define DEF_FREQUENCY_UP_THRESHOLD (80)
19#define DEF_FREQUENCY_DOWN_THRESHOLD (20) 35#define DEF_FREQUENCY_DOWN_THRESHOLD (20)
@@ -21,18 +37,6 @@
21#define DEF_SAMPLING_DOWN_FACTOR (1) 37#define DEF_SAMPLING_DOWN_FACTOR (1)
22#define MAX_SAMPLING_DOWN_FACTOR (10) 38#define MAX_SAMPLING_DOWN_FACTOR (10)
23 39
24static DEFINE_PER_CPU(struct cs_cpu_dbs_info_s, cs_cpu_dbs_info);
25
26static int cs_cpufreq_governor_dbs(struct cpufreq_policy *policy,
27 unsigned int event);
28
29static struct cpufreq_governor cpufreq_gov_conservative = {
30 .name = "conservative",
31 .governor = cs_cpufreq_governor_dbs,
32 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
33 .owner = THIS_MODULE,
34};
35
36static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners, 40static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners,
37 struct cpufreq_policy *policy) 41 struct cpufreq_policy *policy)
38{ 42{
@@ -54,27 +58,28 @@ static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners,
54 * Any frequency increase takes it to the maximum frequency. Frequency reduction 58 * Any frequency increase takes it to the maximum frequency. Frequency reduction
55 * happens at minimum steps of 5% (default) of maximum frequency 59 * happens at minimum steps of 5% (default) of maximum frequency
56 */ 60 */
57static void cs_check_cpu(int cpu, unsigned int load) 61static unsigned int cs_dbs_timer(struct cpufreq_policy *policy)
58{ 62{
59 struct cs_cpu_dbs_info_s *dbs_info = &per_cpu(cs_cpu_dbs_info, cpu); 63 struct policy_dbs_info *policy_dbs = policy->governor_data;
60 struct cpufreq_policy *policy = dbs_info->cdbs.shared->policy; 64 struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
61 struct dbs_data *dbs_data = policy->governor_data; 65 struct dbs_data *dbs_data = policy_dbs->dbs_data;
62 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 66 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
67 unsigned int load = dbs_update(policy);
63 68
64 /* 69 /*
65 * break out if we 'cannot' reduce the speed as the user might 70 * break out if we 'cannot' reduce the speed as the user might
66 * want freq_step to be zero 71 * want freq_step to be zero
67 */ 72 */
68 if (cs_tuners->freq_step == 0) 73 if (cs_tuners->freq_step == 0)
69 return; 74 goto out;
70 75
71 /* Check for frequency increase */ 76 /* Check for frequency increase */
72 if (load > cs_tuners->up_threshold) { 77 if (load > dbs_data->up_threshold) {
73 dbs_info->down_skip = 0; 78 dbs_info->down_skip = 0;
74 79
75 /* if we are already at full speed then break out early */ 80 /* if we are already at full speed then break out early */
76 if (dbs_info->requested_freq == policy->max) 81 if (dbs_info->requested_freq == policy->max)
77 return; 82 goto out;
78 83
79 dbs_info->requested_freq += get_freq_target(cs_tuners, policy); 84 dbs_info->requested_freq += get_freq_target(cs_tuners, policy);
80 85
@@ -83,12 +88,12 @@ static void cs_check_cpu(int cpu, unsigned int load)
83 88
84 __cpufreq_driver_target(policy, dbs_info->requested_freq, 89 __cpufreq_driver_target(policy, dbs_info->requested_freq,
85 CPUFREQ_RELATION_H); 90 CPUFREQ_RELATION_H);
86 return; 91 goto out;
87 } 92 }
88 93
89 /* if sampling_down_factor is active break out early */ 94 /* if sampling_down_factor is active break out early */
90 if (++dbs_info->down_skip < cs_tuners->sampling_down_factor) 95 if (++dbs_info->down_skip < dbs_data->sampling_down_factor)
91 return; 96 goto out;
92 dbs_info->down_skip = 0; 97 dbs_info->down_skip = 0;
93 98
94 /* Check for frequency decrease */ 99 /* Check for frequency decrease */
@@ -98,7 +103,7 @@ static void cs_check_cpu(int cpu, unsigned int load)
98 * if we cannot reduce the frequency anymore, break out early 103 * if we cannot reduce the frequency anymore, break out early
99 */ 104 */
100 if (policy->cur == policy->min) 105 if (policy->cur == policy->min)
101 return; 106 goto out;
102 107
103 freq_target = get_freq_target(cs_tuners, policy); 108 freq_target = get_freq_target(cs_tuners, policy);
104 if (dbs_info->requested_freq > freq_target) 109 if (dbs_info->requested_freq > freq_target)
@@ -108,58 +113,25 @@ static void cs_check_cpu(int cpu, unsigned int load)
108 113
109 __cpufreq_driver_target(policy, dbs_info->requested_freq, 114 __cpufreq_driver_target(policy, dbs_info->requested_freq,
110 CPUFREQ_RELATION_L); 115 CPUFREQ_RELATION_L);
111 return;
112 } 116 }
113}
114
115static unsigned int cs_dbs_timer(struct cpufreq_policy *policy, bool modify_all)
116{
117 struct dbs_data *dbs_data = policy->governor_data;
118 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
119
120 if (modify_all)
121 dbs_check_cpu(dbs_data, policy->cpu);
122 117
123 return delay_for_sampling_rate(cs_tuners->sampling_rate); 118 out:
119 return dbs_data->sampling_rate;
124} 120}
125 121
126static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 122static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
127 void *data) 123 void *data);
128{
129 struct cpufreq_freqs *freq = data;
130 struct cs_cpu_dbs_info_s *dbs_info =
131 &per_cpu(cs_cpu_dbs_info, freq->cpu);
132 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(freq->cpu);
133
134 if (!policy)
135 return 0;
136
137 /* policy isn't governed by conservative governor */
138 if (policy->governor != &cpufreq_gov_conservative)
139 return 0;
140
141 /*
142 * we only care if our internally tracked freq moves outside the 'valid'
143 * ranges of frequency available to us otherwise we do not change it
144 */
145 if (dbs_info->requested_freq > policy->max
146 || dbs_info->requested_freq < policy->min)
147 dbs_info->requested_freq = freq->new;
148
149 return 0;
150}
151 124
152static struct notifier_block cs_cpufreq_notifier_block = { 125static struct notifier_block cs_cpufreq_notifier_block = {
153 .notifier_call = dbs_cpufreq_notifier, 126 .notifier_call = dbs_cpufreq_notifier,
154}; 127};
155 128
156/************************** sysfs interface ************************/ 129/************************** sysfs interface ************************/
157static struct common_dbs_data cs_dbs_cdata; 130static struct dbs_governor cs_dbs_gov;
158 131
159static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data, 132static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data,
160 const char *buf, size_t count) 133 const char *buf, size_t count)
161{ 134{
162 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
163 unsigned int input; 135 unsigned int input;
164 int ret; 136 int ret;
165 ret = sscanf(buf, "%u", &input); 137 ret = sscanf(buf, "%u", &input);
@@ -167,22 +139,7 @@ static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data,
167 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) 139 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
168 return -EINVAL; 140 return -EINVAL;
169 141
170 cs_tuners->sampling_down_factor = input; 142 dbs_data->sampling_down_factor = input;
171 return count;
172}
173
174static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf,
175 size_t count)
176{
177 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
178 unsigned int input;
179 int ret;
180 ret = sscanf(buf, "%u", &input);
181
182 if (ret != 1)
183 return -EINVAL;
184
185 cs_tuners->sampling_rate = max(input, dbs_data->min_sampling_rate);
186 return count; 143 return count;
187} 144}
188 145
@@ -197,7 +154,7 @@ static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf,
197 if (ret != 1 || input > 100 || input <= cs_tuners->down_threshold) 154 if (ret != 1 || input > 100 || input <= cs_tuners->down_threshold)
198 return -EINVAL; 155 return -EINVAL;
199 156
200 cs_tuners->up_threshold = input; 157 dbs_data->up_threshold = input;
201 return count; 158 return count;
202} 159}
203 160
@@ -211,7 +168,7 @@ static ssize_t store_down_threshold(struct dbs_data *dbs_data, const char *buf,
211 168
212 /* cannot be lower than 11 otherwise freq will not fall */ 169 /* cannot be lower than 11 otherwise freq will not fall */
213 if (ret != 1 || input < 11 || input > 100 || 170 if (ret != 1 || input < 11 || input > 100 ||
214 input >= cs_tuners->up_threshold) 171 input >= dbs_data->up_threshold)
215 return -EINVAL; 172 return -EINVAL;
216 173
217 cs_tuners->down_threshold = input; 174 cs_tuners->down_threshold = input;
@@ -221,8 +178,7 @@ static ssize_t store_down_threshold(struct dbs_data *dbs_data, const char *buf,
221static ssize_t store_ignore_nice_load(struct dbs_data *dbs_data, 178static ssize_t store_ignore_nice_load(struct dbs_data *dbs_data,
222 const char *buf, size_t count) 179 const char *buf, size_t count)
223{ 180{
224 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 181 unsigned int input;
225 unsigned int input, j;
226 int ret; 182 int ret;
227 183
228 ret = sscanf(buf, "%u", &input); 184 ret = sscanf(buf, "%u", &input);
@@ -232,21 +188,14 @@ static ssize_t store_ignore_nice_load(struct dbs_data *dbs_data,
232 if (input > 1) 188 if (input > 1)
233 input = 1; 189 input = 1;
234 190
235 if (input == cs_tuners->ignore_nice_load) /* nothing to do */ 191 if (input == dbs_data->ignore_nice_load) /* nothing to do */
236 return count; 192 return count;
237 193
238 cs_tuners->ignore_nice_load = input; 194 dbs_data->ignore_nice_load = input;
239 195
240 /* we need to re-evaluate prev_cpu_idle */ 196 /* we need to re-evaluate prev_cpu_idle */
241 for_each_online_cpu(j) { 197 gov_update_cpu_data(dbs_data);
242 struct cs_cpu_dbs_info_s *dbs_info; 198
243 dbs_info = &per_cpu(cs_cpu_dbs_info, j);
244 dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j,
245 &dbs_info->cdbs.prev_cpu_wall, 0);
246 if (cs_tuners->ignore_nice_load)
247 dbs_info->cdbs.prev_cpu_nice =
248 kcpustat_cpu(j).cpustat[CPUTIME_NICE];
249 }
250 return count; 199 return count;
251} 200}
252 201
@@ -272,55 +221,47 @@ static ssize_t store_freq_step(struct dbs_data *dbs_data, const char *buf,
272 return count; 221 return count;
273} 222}
274 223
275show_store_one(cs, sampling_rate); 224gov_show_one_common(sampling_rate);
276show_store_one(cs, sampling_down_factor); 225gov_show_one_common(sampling_down_factor);
277show_store_one(cs, up_threshold); 226gov_show_one_common(up_threshold);
278show_store_one(cs, down_threshold); 227gov_show_one_common(ignore_nice_load);
279show_store_one(cs, ignore_nice_load); 228gov_show_one_common(min_sampling_rate);
280show_store_one(cs, freq_step); 229gov_show_one(cs, down_threshold);
281declare_show_sampling_rate_min(cs); 230gov_show_one(cs, freq_step);
282 231
283gov_sys_pol_attr_rw(sampling_rate); 232gov_attr_rw(sampling_rate);
284gov_sys_pol_attr_rw(sampling_down_factor); 233gov_attr_rw(sampling_down_factor);
285gov_sys_pol_attr_rw(up_threshold); 234gov_attr_rw(up_threshold);
286gov_sys_pol_attr_rw(down_threshold); 235gov_attr_rw(ignore_nice_load);
287gov_sys_pol_attr_rw(ignore_nice_load); 236gov_attr_ro(min_sampling_rate);
288gov_sys_pol_attr_rw(freq_step); 237gov_attr_rw(down_threshold);
289gov_sys_pol_attr_ro(sampling_rate_min); 238gov_attr_rw(freq_step);
290 239
291static struct attribute *dbs_attributes_gov_sys[] = { 240static struct attribute *cs_attributes[] = {
292 &sampling_rate_min_gov_sys.attr, 241 &min_sampling_rate.attr,
293 &sampling_rate_gov_sys.attr, 242 &sampling_rate.attr,
294 &sampling_down_factor_gov_sys.attr, 243 &sampling_down_factor.attr,
295 &up_threshold_gov_sys.attr, 244 &up_threshold.attr,
296 &down_threshold_gov_sys.attr, 245 &down_threshold.attr,
297 &ignore_nice_load_gov_sys.attr, 246 &ignore_nice_load.attr,
298 &freq_step_gov_sys.attr, 247 &freq_step.attr,
299 NULL 248 NULL
300}; 249};
301 250
302static struct attribute_group cs_attr_group_gov_sys = { 251/************************** sysfs end ************************/
303 .attrs = dbs_attributes_gov_sys,
304 .name = "conservative",
305};
306 252
307static struct attribute *dbs_attributes_gov_pol[] = { 253static struct policy_dbs_info *cs_alloc(void)
308 &sampling_rate_min_gov_pol.attr, 254{
309 &sampling_rate_gov_pol.attr, 255 struct cs_policy_dbs_info *dbs_info;
310 &sampling_down_factor_gov_pol.attr,
311 &up_threshold_gov_pol.attr,
312 &down_threshold_gov_pol.attr,
313 &ignore_nice_load_gov_pol.attr,
314 &freq_step_gov_pol.attr,
315 NULL
316};
317 256
318static struct attribute_group cs_attr_group_gov_pol = { 257 dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL);
319 .attrs = dbs_attributes_gov_pol, 258 return dbs_info ? &dbs_info->policy_dbs : NULL;
320 .name = "conservative", 259}
321};
322 260
323/************************** sysfs end ************************/ 261static void cs_free(struct policy_dbs_info *policy_dbs)
262{
263 kfree(to_dbs_info(policy_dbs));
264}
324 265
325static int cs_init(struct dbs_data *dbs_data, bool notify) 266static int cs_init(struct dbs_data *dbs_data, bool notify)
326{ 267{
@@ -332,11 +273,11 @@ static int cs_init(struct dbs_data *dbs_data, bool notify)
332 return -ENOMEM; 273 return -ENOMEM;
333 } 274 }
334 275
335 tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
336 tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD; 276 tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD;
337 tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
338 tuners->ignore_nice_load = 0;
339 tuners->freq_step = DEF_FREQUENCY_STEP; 277 tuners->freq_step = DEF_FREQUENCY_STEP;
278 dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
279 dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
280 dbs_data->ignore_nice_load = 0;
340 281
341 dbs_data->tuners = tuners; 282 dbs_data->tuners = tuners;
342 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * 283 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
@@ -358,35 +299,66 @@ static void cs_exit(struct dbs_data *dbs_data, bool notify)
358 kfree(dbs_data->tuners); 299 kfree(dbs_data->tuners);
359} 300}
360 301
361define_get_cpu_dbs_routines(cs_cpu_dbs_info); 302static void cs_start(struct cpufreq_policy *policy)
303{
304 struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
305
306 dbs_info->down_skip = 0;
307 dbs_info->requested_freq = policy->cur;
308}
362 309
363static struct common_dbs_data cs_dbs_cdata = { 310static struct dbs_governor cs_dbs_gov = {
364 .governor = GOV_CONSERVATIVE, 311 .gov = {
365 .attr_group_gov_sys = &cs_attr_group_gov_sys, 312 .name = "conservative",
366 .attr_group_gov_pol = &cs_attr_group_gov_pol, 313 .governor = cpufreq_governor_dbs,
367 .get_cpu_cdbs = get_cpu_cdbs, 314 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
368 .get_cpu_dbs_info_s = get_cpu_dbs_info_s, 315 .owner = THIS_MODULE,
316 },
317 .kobj_type = { .default_attrs = cs_attributes },
369 .gov_dbs_timer = cs_dbs_timer, 318 .gov_dbs_timer = cs_dbs_timer,
370 .gov_check_cpu = cs_check_cpu, 319 .alloc = cs_alloc,
320 .free = cs_free,
371 .init = cs_init, 321 .init = cs_init,
372 .exit = cs_exit, 322 .exit = cs_exit,
373 .mutex = __MUTEX_INITIALIZER(cs_dbs_cdata.mutex), 323 .start = cs_start,
374}; 324};
375 325
376static int cs_cpufreq_governor_dbs(struct cpufreq_policy *policy, 326#define CPU_FREQ_GOV_CONSERVATIVE (&cs_dbs_gov.gov)
377 unsigned int event) 327
328static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
329 void *data)
378{ 330{
379 return cpufreq_governor_dbs(policy, &cs_dbs_cdata, event); 331 struct cpufreq_freqs *freq = data;
332 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(freq->cpu);
333 struct cs_policy_dbs_info *dbs_info;
334
335 if (!policy)
336 return 0;
337
338 /* policy isn't governed by conservative governor */
339 if (policy->governor != CPU_FREQ_GOV_CONSERVATIVE)
340 return 0;
341
342 dbs_info = to_dbs_info(policy->governor_data);
343 /*
344 * we only care if our internally tracked freq moves outside the 'valid'
345 * ranges of frequency available to us otherwise we do not change it
346 */
347 if (dbs_info->requested_freq > policy->max
348 || dbs_info->requested_freq < policy->min)
349 dbs_info->requested_freq = freq->new;
350
351 return 0;
380} 352}
381 353
382static int __init cpufreq_gov_dbs_init(void) 354static int __init cpufreq_gov_dbs_init(void)
383{ 355{
384 return cpufreq_register_governor(&cpufreq_gov_conservative); 356 return cpufreq_register_governor(CPU_FREQ_GOV_CONSERVATIVE);
385} 357}
386 358
387static void __exit cpufreq_gov_dbs_exit(void) 359static void __exit cpufreq_gov_dbs_exit(void)
388{ 360{
389 cpufreq_unregister_governor(&cpufreq_gov_conservative); 361 cpufreq_unregister_governor(CPU_FREQ_GOV_CONSERVATIVE);
390} 362}
391 363
392MODULE_AUTHOR("Alexander Clouter <alex@digriz.org.uk>"); 364MODULE_AUTHOR("Alexander Clouter <alex@digriz.org.uk>");
@@ -398,7 +370,7 @@ MODULE_LICENSE("GPL");
398#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE 370#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE
399struct cpufreq_governor *cpufreq_default_governor(void) 371struct cpufreq_governor *cpufreq_default_governor(void)
400{ 372{
401 return &cpufreq_gov_conservative; 373 return CPU_FREQ_GOV_CONSERVATIVE;
402} 374}
403 375
404fs_initcall(cpufreq_gov_dbs_init); 376fs_initcall(cpufreq_gov_dbs_init);
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index e0d111024d48..1c25ef405616 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -18,95 +18,193 @@
18 18
19#include <linux/export.h> 19#include <linux/export.h>
20#include <linux/kernel_stat.h> 20#include <linux/kernel_stat.h>
21#include <linux/sched.h>
21#include <linux/slab.h> 22#include <linux/slab.h>
22 23
23#include "cpufreq_governor.h" 24#include "cpufreq_governor.h"
24 25
25static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data) 26static DEFINE_PER_CPU(struct cpu_dbs_info, cpu_dbs);
26{ 27
27 if (have_governor_per_policy()) 28static DEFINE_MUTEX(gov_dbs_data_mutex);
28 return dbs_data->cdata->attr_group_gov_pol;
29 else
30 return dbs_data->cdata->attr_group_gov_sys;
31}
32 29
33void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) 30/* Common sysfs tunables */
31/**
32 * store_sampling_rate - update sampling rate effective immediately if needed.
33 *
34 * If new rate is smaller than the old, simply updating
35 * dbs.sampling_rate might not be appropriate. For example, if the
36 * original sampling_rate was 1 second and the requested new sampling rate is 10
37 * ms because the user needs immediate reaction from ondemand governor, but not
38 * sure if higher frequency will be required or not, then, the governor may
39 * change the sampling rate too late; up to 1 second later. Thus, if we are
40 * reducing the sampling rate, we need to make the new value effective
41 * immediately.
42 *
43 * This must be called with dbs_data->mutex held, otherwise traversing
44 * policy_dbs_list isn't safe.
45 */
46ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf,
47 size_t count)
34{ 48{
35 struct cpu_dbs_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); 49 struct policy_dbs_info *policy_dbs;
36 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 50 unsigned int rate;
37 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 51 int ret;
38 struct cpufreq_policy *policy = cdbs->shared->policy; 52 ret = sscanf(buf, "%u", &rate);
39 unsigned int sampling_rate; 53 if (ret != 1)
40 unsigned int max_load = 0; 54 return -EINVAL;
41 unsigned int ignore_nice;
42 unsigned int j;
43 55
44 if (dbs_data->cdata->governor == GOV_ONDEMAND) { 56 dbs_data->sampling_rate = max(rate, dbs_data->min_sampling_rate);
45 struct od_cpu_dbs_info_s *od_dbs_info =
46 dbs_data->cdata->get_cpu_dbs_info_s(cpu);
47 57
58 /*
59 * We are operating under dbs_data->mutex and so the list and its
60 * entries can't be freed concurrently.
61 */
62 list_for_each_entry(policy_dbs, &dbs_data->policy_dbs_list, list) {
63 mutex_lock(&policy_dbs->timer_mutex);
48 /* 64 /*
49 * Sometimes, the ondemand governor uses an additional 65 * On 32-bit architectures this may race with the
50 * multiplier to give long delays. So apply this multiplier to 66 * sample_delay_ns read in dbs_update_util_handler(), but that
51 * the 'sampling_rate', so as to keep the wake-up-from-idle 67 * really doesn't matter. If the read returns a value that's
52 * detection logic a bit conservative. 68 * too big, the sample will be skipped, but the next invocation
69 * of dbs_update_util_handler() (when the update has been
70 * completed) will take a sample.
71 *
72 * If this runs in parallel with dbs_work_handler(), we may end
73 * up overwriting the sample_delay_ns value that it has just
74 * written, but it will be corrected next time a sample is
75 * taken, so it shouldn't be significant.
53 */ 76 */
54 sampling_rate = od_tuners->sampling_rate; 77 gov_update_sample_delay(policy_dbs, 0);
55 sampling_rate *= od_dbs_info->rate_mult; 78 mutex_unlock(&policy_dbs->timer_mutex);
79 }
56 80
57 ignore_nice = od_tuners->ignore_nice_load; 81 return count;
58 } else { 82}
59 sampling_rate = cs_tuners->sampling_rate; 83EXPORT_SYMBOL_GPL(store_sampling_rate);
60 ignore_nice = cs_tuners->ignore_nice_load; 84
85/**
86 * gov_update_cpu_data - Update CPU load data.
87 * @dbs_data: Top-level governor data pointer.
88 *
89 * Update CPU load data for all CPUs in the domain governed by @dbs_data
90 * (that may be a single policy or a bunch of them if governor tunables are
91 * system-wide).
92 *
93 * Call under the @dbs_data mutex.
94 */
95void gov_update_cpu_data(struct dbs_data *dbs_data)
96{
97 struct policy_dbs_info *policy_dbs;
98
99 list_for_each_entry(policy_dbs, &dbs_data->policy_dbs_list, list) {
100 unsigned int j;
101
102 for_each_cpu(j, policy_dbs->policy->cpus) {
103 struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
104
105 j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall,
106 dbs_data->io_is_busy);
107 if (dbs_data->ignore_nice_load)
108 j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
109 }
61 } 110 }
111}
112EXPORT_SYMBOL_GPL(gov_update_cpu_data);
113
114static inline struct dbs_data *to_dbs_data(struct kobject *kobj)
115{
116 return container_of(kobj, struct dbs_data, kobj);
117}
118
119static inline struct governor_attr *to_gov_attr(struct attribute *attr)
120{
121 return container_of(attr, struct governor_attr, attr);
122}
123
124static ssize_t governor_show(struct kobject *kobj, struct attribute *attr,
125 char *buf)
126{
127 struct dbs_data *dbs_data = to_dbs_data(kobj);
128 struct governor_attr *gattr = to_gov_attr(attr);
129
130 return gattr->show(dbs_data, buf);
131}
132
133static ssize_t governor_store(struct kobject *kobj, struct attribute *attr,
134 const char *buf, size_t count)
135{
136 struct dbs_data *dbs_data = to_dbs_data(kobj);
137 struct governor_attr *gattr = to_gov_attr(attr);
138 int ret = -EBUSY;
139
140 mutex_lock(&dbs_data->mutex);
141
142 if (dbs_data->usage_count)
143 ret = gattr->store(dbs_data, buf, count);
144
145 mutex_unlock(&dbs_data->mutex);
146
147 return ret;
148}
149
150/*
151 * Sysfs Ops for accessing governor attributes.
152 *
153 * All show/store invocations for governor specific sysfs attributes, will first
154 * call the below show/store callbacks and the attribute specific callback will
155 * be called from within it.
156 */
157static const struct sysfs_ops governor_sysfs_ops = {
158 .show = governor_show,
159 .store = governor_store,
160};
161
162unsigned int dbs_update(struct cpufreq_policy *policy)
163{
164 struct policy_dbs_info *policy_dbs = policy->governor_data;
165 struct dbs_data *dbs_data = policy_dbs->dbs_data;
166 unsigned int ignore_nice = dbs_data->ignore_nice_load;
167 unsigned int max_load = 0;
168 unsigned int sampling_rate, io_busy, j;
169
170 /*
171 * Sometimes governors may use an additional multiplier to increase
172 * sample delays temporarily. Apply that multiplier to sampling_rate
173 * so as to keep the wake-up-from-idle detection logic a bit
174 * conservative.
175 */
176 sampling_rate = dbs_data->sampling_rate * policy_dbs->rate_mult;
177 /*
178 * For the purpose of ondemand, waiting for disk IO is an indication
179 * that you're performance critical, and not that the system is actually
180 * idle, so do not add the iowait time to the CPU idle time then.
181 */
182 io_busy = dbs_data->io_is_busy;
62 183
63 /* Get Absolute Load */ 184 /* Get Absolute Load */
64 for_each_cpu(j, policy->cpus) { 185 for_each_cpu(j, policy->cpus) {
65 struct cpu_dbs_info *j_cdbs; 186 struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
66 u64 cur_wall_time, cur_idle_time; 187 u64 cur_wall_time, cur_idle_time;
67 unsigned int idle_time, wall_time; 188 unsigned int idle_time, wall_time;
68 unsigned int load; 189 unsigned int load;
69 int io_busy = 0;
70
71 j_cdbs = dbs_data->cdata->get_cpu_cdbs(j);
72 190
73 /*
74 * For the purpose of ondemand, waiting for disk IO is
75 * an indication that you're performance critical, and
76 * not that the system is actually idle. So do not add
77 * the iowait time to the cpu idle time.
78 */
79 if (dbs_data->cdata->governor == GOV_ONDEMAND)
80 io_busy = od_tuners->io_is_busy;
81 cur_idle_time = get_cpu_idle_time(j, &cur_wall_time, io_busy); 191 cur_idle_time = get_cpu_idle_time(j, &cur_wall_time, io_busy);
82 192
83 wall_time = (unsigned int) 193 wall_time = cur_wall_time - j_cdbs->prev_cpu_wall;
84 (cur_wall_time - j_cdbs->prev_cpu_wall);
85 j_cdbs->prev_cpu_wall = cur_wall_time; 194 j_cdbs->prev_cpu_wall = cur_wall_time;
86 195
87 if (cur_idle_time < j_cdbs->prev_cpu_idle) 196 if (cur_idle_time <= j_cdbs->prev_cpu_idle) {
88 cur_idle_time = j_cdbs->prev_cpu_idle; 197 idle_time = 0;
89 198 } else {
90 idle_time = (unsigned int) 199 idle_time = cur_idle_time - j_cdbs->prev_cpu_idle;
91 (cur_idle_time - j_cdbs->prev_cpu_idle); 200 j_cdbs->prev_cpu_idle = cur_idle_time;
92 j_cdbs->prev_cpu_idle = cur_idle_time; 201 }
93 202
94 if (ignore_nice) { 203 if (ignore_nice) {
95 u64 cur_nice; 204 u64 cur_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
96 unsigned long cur_nice_jiffies;
97
98 cur_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE] -
99 cdbs->prev_cpu_nice;
100 /*
101 * Assumption: nice time between sampling periods will
102 * be less than 2^32 jiffies for 32 bit sys
103 */
104 cur_nice_jiffies = (unsigned long)
105 cputime64_to_jiffies64(cur_nice);
106 205
107 cdbs->prev_cpu_nice = 206 idle_time += cputime_to_usecs(cur_nice - j_cdbs->prev_cpu_nice);
108 kcpustat_cpu(j).cpustat[CPUTIME_NICE]; 207 j_cdbs->prev_cpu_nice = cur_nice;
109 idle_time += jiffies_to_usecs(cur_nice_jiffies);
110 } 208 }
111 209
112 if (unlikely(!wall_time || wall_time < idle_time)) 210 if (unlikely(!wall_time || wall_time < idle_time))
@@ -128,10 +226,10 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
128 * dropped down. So we perform the copy only once, upon the 226 * dropped down. So we perform the copy only once, upon the
129 * first wake-up from idle.) 227 * first wake-up from idle.)
130 * 228 *
131 * Detecting this situation is easy: the governor's deferrable 229 * Detecting this situation is easy: the governor's utilization
132 * timer would not have fired during CPU-idle periods. Hence 230 * update handler would not have run during CPU-idle periods.
133 * an unusually large 'wall_time' (as compared to the sampling 231 * Hence, an unusually large 'wall_time' (as compared to the
134 * rate) indicates this scenario. 232 * sampling rate) indicates this scenario.
135 * 233 *
136 * prev_load can be zero in two cases and we must recalculate it 234 * prev_load can be zero in two cases and we must recalculate it
137 * for both cases: 235 * for both cases:
@@ -156,222 +254,224 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
156 if (load > max_load) 254 if (load > max_load)
157 max_load = load; 255 max_load = load;
158 } 256 }
159 257 return max_load;
160 dbs_data->cdata->gov_check_cpu(cpu, max_load);
161} 258}
162EXPORT_SYMBOL_GPL(dbs_check_cpu); 259EXPORT_SYMBOL_GPL(dbs_update);
163 260
164void gov_add_timers(struct cpufreq_policy *policy, unsigned int delay) 261static void gov_set_update_util(struct policy_dbs_info *policy_dbs,
262 unsigned int delay_us)
165{ 263{
166 struct dbs_data *dbs_data = policy->governor_data; 264 struct cpufreq_policy *policy = policy_dbs->policy;
167 struct cpu_dbs_info *cdbs;
168 int cpu; 265 int cpu;
169 266
267 gov_update_sample_delay(policy_dbs, delay_us);
268 policy_dbs->last_sample_time = 0;
269
170 for_each_cpu(cpu, policy->cpus) { 270 for_each_cpu(cpu, policy->cpus) {
171 cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); 271 struct cpu_dbs_info *cdbs = &per_cpu(cpu_dbs, cpu);
172 cdbs->timer.expires = jiffies + delay; 272
173 add_timer_on(&cdbs->timer, cpu); 273 cpufreq_set_update_util_data(cpu, &cdbs->update_util);
174 } 274 }
175} 275}
176EXPORT_SYMBOL_GPL(gov_add_timers);
177 276
178static inline void gov_cancel_timers(struct cpufreq_policy *policy) 277static inline void gov_clear_update_util(struct cpufreq_policy *policy)
179{ 278{
180 struct dbs_data *dbs_data = policy->governor_data;
181 struct cpu_dbs_info *cdbs;
182 int i; 279 int i;
183 280
184 for_each_cpu(i, policy->cpus) { 281 for_each_cpu(i, policy->cpus)
185 cdbs = dbs_data->cdata->get_cpu_cdbs(i); 282 cpufreq_set_update_util_data(i, NULL);
186 del_timer_sync(&cdbs->timer);
187 }
188}
189 283
190void gov_cancel_work(struct cpu_common_dbs_info *shared) 284 synchronize_sched();
191{
192 /* Tell dbs_timer_handler() to skip queuing up work items. */
193 atomic_inc(&shared->skip_work);
194 /*
195 * If dbs_timer_handler() is already running, it may not notice the
196 * incremented skip_work, so wait for it to complete to prevent its work
197 * item from being queued up after the cancel_work_sync() below.
198 */
199 gov_cancel_timers(shared->policy);
200 /*
201 * In case dbs_timer_handler() managed to run and spawn a work item
202 * before the timers have been canceled, wait for that work item to
203 * complete and then cancel all of the timers set up by it. If
204 * dbs_timer_handler() runs again at that point, it will see the
205 * positive value of skip_work and won't spawn any more work items.
206 */
207 cancel_work_sync(&shared->work);
208 gov_cancel_timers(shared->policy);
209 atomic_set(&shared->skip_work, 0);
210} 285}
211EXPORT_SYMBOL_GPL(gov_cancel_work);
212 286
213/* Will return if we need to evaluate cpu load again or not */ 287static void gov_cancel_work(struct cpufreq_policy *policy)
214static bool need_load_eval(struct cpu_common_dbs_info *shared,
215 unsigned int sampling_rate)
216{ 288{
217 if (policy_is_shared(shared->policy)) { 289 struct policy_dbs_info *policy_dbs = policy->governor_data;
218 ktime_t time_now = ktime_get();
219 s64 delta_us = ktime_us_delta(time_now, shared->time_stamp);
220
221 /* Do nothing if we recently have sampled */
222 if (delta_us < (s64)(sampling_rate / 2))
223 return false;
224 else
225 shared->time_stamp = time_now;
226 }
227 290
228 return true; 291 gov_clear_update_util(policy_dbs->policy);
292 irq_work_sync(&policy_dbs->irq_work);
293 cancel_work_sync(&policy_dbs->work);
294 atomic_set(&policy_dbs->work_count, 0);
295 policy_dbs->work_in_progress = false;
229} 296}
230 297
231static void dbs_work_handler(struct work_struct *work) 298static void dbs_work_handler(struct work_struct *work)
232{ 299{
233 struct cpu_common_dbs_info *shared = container_of(work, struct 300 struct policy_dbs_info *policy_dbs;
234 cpu_common_dbs_info, work);
235 struct cpufreq_policy *policy; 301 struct cpufreq_policy *policy;
236 struct dbs_data *dbs_data; 302 struct dbs_governor *gov;
237 unsigned int sampling_rate, delay;
238 bool eval_load;
239
240 policy = shared->policy;
241 dbs_data = policy->governor_data;
242 303
243 /* Kill all timers */ 304 policy_dbs = container_of(work, struct policy_dbs_info, work);
244 gov_cancel_timers(policy); 305 policy = policy_dbs->policy;
306 gov = dbs_governor_of(policy);
245 307
246 if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { 308 /*
247 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 309 * Make sure cpufreq_governor_limits() isn't evaluating load or the
248 310 * ondemand governor isn't updating the sampling rate in parallel.
249 sampling_rate = cs_tuners->sampling_rate; 311 */
250 } else { 312 mutex_lock(&policy_dbs->timer_mutex);
251 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 313 gov_update_sample_delay(policy_dbs, gov->gov_dbs_timer(policy));
252 314 mutex_unlock(&policy_dbs->timer_mutex);
253 sampling_rate = od_tuners->sampling_rate;
254 }
255
256 eval_load = need_load_eval(shared, sampling_rate);
257 315
316 /* Allow the utilization update handler to queue up more work. */
317 atomic_set(&policy_dbs->work_count, 0);
258 /* 318 /*
259 * Make sure cpufreq_governor_limits() isn't evaluating load in 319 * If the update below is reordered with respect to the sample delay
260 * parallel. 320 * modification, the utilization update handler may end up using a stale
321 * sample delay value.
261 */ 322 */
262 mutex_lock(&shared->timer_mutex); 323 smp_wmb();
263 delay = dbs_data->cdata->gov_dbs_timer(policy, eval_load); 324 policy_dbs->work_in_progress = false;
264 mutex_unlock(&shared->timer_mutex); 325}
265 326
266 atomic_dec(&shared->skip_work); 327static void dbs_irq_work(struct irq_work *irq_work)
328{
329 struct policy_dbs_info *policy_dbs;
267 330
268 gov_add_timers(policy, delay); 331 policy_dbs = container_of(irq_work, struct policy_dbs_info, irq_work);
332 schedule_work(&policy_dbs->work);
269} 333}
270 334
271static void dbs_timer_handler(unsigned long data) 335static void dbs_update_util_handler(struct update_util_data *data, u64 time,
336 unsigned long util, unsigned long max)
272{ 337{
273 struct cpu_dbs_info *cdbs = (struct cpu_dbs_info *)data; 338 struct cpu_dbs_info *cdbs = container_of(data, struct cpu_dbs_info, update_util);
274 struct cpu_common_dbs_info *shared = cdbs->shared; 339 struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
340 u64 delta_ns, lst;
275 341
276 /* 342 /*
277 * Timer handler may not be allowed to queue the work at the moment, 343 * The work may not be allowed to be queued up right now.
278 * because: 344 * Possible reasons:
279 * - Another timer handler has done that 345 * - Work has already been queued up or is in progress.
280 * - We are stopping the governor 346 * - It is too early (too little time from the previous sample).
281 * - Or we are updating the sampling rate of the ondemand governor
282 */ 347 */
283 if (atomic_inc_return(&shared->skip_work) > 1) 348 if (policy_dbs->work_in_progress)
284 atomic_dec(&shared->skip_work); 349 return;
285 else
286 queue_work(system_wq, &shared->work);
287}
288 350
289static void set_sampling_rate(struct dbs_data *dbs_data, 351 /*
290 unsigned int sampling_rate) 352 * If the reads below are reordered before the check above, the value
291{ 353 * of sample_delay_ns used in the computation may be stale.
292 if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { 354 */
293 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 355 smp_rmb();
294 cs_tuners->sampling_rate = sampling_rate; 356 lst = READ_ONCE(policy_dbs->last_sample_time);
295 } else { 357 delta_ns = time - lst;
296 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 358 if ((s64)delta_ns < policy_dbs->sample_delay_ns)
297 od_tuners->sampling_rate = sampling_rate; 359 return;
360
361 /*
362 * If the policy is not shared, the irq_work may be queued up right away
363 * at this point. Otherwise, we need to ensure that only one of the
364 * CPUs sharing the policy will do that.
365 */
366 if (policy_dbs->is_shared) {
367 if (!atomic_add_unless(&policy_dbs->work_count, 1, 1))
368 return;
369
370 /*
371 * If another CPU updated last_sample_time in the meantime, we
372 * shouldn't be here, so clear the work counter and bail out.
373 */
374 if (unlikely(lst != READ_ONCE(policy_dbs->last_sample_time))) {
375 atomic_set(&policy_dbs->work_count, 0);
376 return;
377 }
298 } 378 }
379
380 policy_dbs->last_sample_time = time;
381 policy_dbs->work_in_progress = true;
382 irq_work_queue(&policy_dbs->irq_work);
299} 383}
300 384
301static int alloc_common_dbs_info(struct cpufreq_policy *policy, 385static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy,
302 struct common_dbs_data *cdata) 386 struct dbs_governor *gov)
303{ 387{
304 struct cpu_common_dbs_info *shared; 388 struct policy_dbs_info *policy_dbs;
305 int j; 389 int j;
306 390
307 /* Allocate memory for the common information for policy->cpus */ 391 /* Allocate memory for per-policy governor data. */
308 shared = kzalloc(sizeof(*shared), GFP_KERNEL); 392 policy_dbs = gov->alloc();
309 if (!shared) 393 if (!policy_dbs)
310 return -ENOMEM; 394 return NULL;
311 395
312 /* Set shared for all CPUs, online+offline */ 396 policy_dbs->policy = policy;
313 for_each_cpu(j, policy->related_cpus) 397 mutex_init(&policy_dbs->timer_mutex);
314 cdata->get_cpu_cdbs(j)->shared = shared; 398 atomic_set(&policy_dbs->work_count, 0);
399 init_irq_work(&policy_dbs->irq_work, dbs_irq_work);
400 INIT_WORK(&policy_dbs->work, dbs_work_handler);
315 401
316 mutex_init(&shared->timer_mutex); 402 /* Set policy_dbs for all CPUs, online+offline */
317 atomic_set(&shared->skip_work, 0); 403 for_each_cpu(j, policy->related_cpus) {
318 INIT_WORK(&shared->work, dbs_work_handler); 404 struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
319 return 0; 405
406 j_cdbs->policy_dbs = policy_dbs;
407 j_cdbs->update_util.func = dbs_update_util_handler;
408 }
409 return policy_dbs;
320} 410}
321 411
322static void free_common_dbs_info(struct cpufreq_policy *policy, 412static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs,
323 struct common_dbs_data *cdata) 413 struct dbs_governor *gov)
324{ 414{
325 struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(policy->cpu);
326 struct cpu_common_dbs_info *shared = cdbs->shared;
327 int j; 415 int j;
328 416
329 mutex_destroy(&shared->timer_mutex); 417 mutex_destroy(&policy_dbs->timer_mutex);
330 418
331 for_each_cpu(j, policy->cpus) 419 for_each_cpu(j, policy_dbs->policy->related_cpus) {
332 cdata->get_cpu_cdbs(j)->shared = NULL; 420 struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
333 421
334 kfree(shared); 422 j_cdbs->policy_dbs = NULL;
423 j_cdbs->update_util.func = NULL;
424 }
425 gov->free(policy_dbs);
335} 426}
336 427
337static int cpufreq_governor_init(struct cpufreq_policy *policy, 428static int cpufreq_governor_init(struct cpufreq_policy *policy)
338 struct dbs_data *dbs_data,
339 struct common_dbs_data *cdata)
340{ 429{
430 struct dbs_governor *gov = dbs_governor_of(policy);
431 struct dbs_data *dbs_data;
432 struct policy_dbs_info *policy_dbs;
341 unsigned int latency; 433 unsigned int latency;
342 int ret; 434 int ret = 0;
343 435
344 /* State should be equivalent to EXIT */ 436 /* State should be equivalent to EXIT */
345 if (policy->governor_data) 437 if (policy->governor_data)
346 return -EBUSY; 438 return -EBUSY;
347 439
348 if (dbs_data) { 440 policy_dbs = alloc_policy_dbs_info(policy, gov);
349 if (WARN_ON(have_governor_per_policy())) 441 if (!policy_dbs)
350 return -EINVAL; 442 return -ENOMEM;
351 443
352 ret = alloc_common_dbs_info(policy, cdata); 444 /* Protect gov->gdbs_data against concurrent updates. */
353 if (ret) 445 mutex_lock(&gov_dbs_data_mutex);
354 return ret;
355 446
447 dbs_data = gov->gdbs_data;
448 if (dbs_data) {
449 if (WARN_ON(have_governor_per_policy())) {
450 ret = -EINVAL;
451 goto free_policy_dbs_info;
452 }
453 policy_dbs->dbs_data = dbs_data;
454 policy->governor_data = policy_dbs;
455
456 mutex_lock(&dbs_data->mutex);
356 dbs_data->usage_count++; 457 dbs_data->usage_count++;
357 policy->governor_data = dbs_data; 458 list_add(&policy_dbs->list, &dbs_data->policy_dbs_list);
358 return 0; 459 mutex_unlock(&dbs_data->mutex);
460 goto out;
359 } 461 }
360 462
361 dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL); 463 dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL);
362 if (!dbs_data) 464 if (!dbs_data) {
363 return -ENOMEM; 465 ret = -ENOMEM;
364 466 goto free_policy_dbs_info;
365 ret = alloc_common_dbs_info(policy, cdata); 467 }
366 if (ret)
367 goto free_dbs_data;
368 468
369 dbs_data->cdata = cdata; 469 INIT_LIST_HEAD(&dbs_data->policy_dbs_list);
370 dbs_data->usage_count = 1; 470 mutex_init(&dbs_data->mutex);
371 471
372 ret = cdata->init(dbs_data, !policy->governor->initialized); 472 ret = gov->init(dbs_data, !policy->governor->initialized);
373 if (ret) 473 if (ret)
374 goto free_common_dbs_info; 474 goto free_policy_dbs_info;
375 475
376 /* policy latency is in ns. Convert it to us first */ 476 /* policy latency is in ns. Convert it to us first */
377 latency = policy->cpuinfo.transition_latency / 1000; 477 latency = policy->cpuinfo.transition_latency / 1000;
@@ -381,216 +481,156 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy,
381 /* Bring kernel and HW constraints together */ 481 /* Bring kernel and HW constraints together */
382 dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate, 482 dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate,
383 MIN_LATENCY_MULTIPLIER * latency); 483 MIN_LATENCY_MULTIPLIER * latency);
384 set_sampling_rate(dbs_data, max(dbs_data->min_sampling_rate, 484 dbs_data->sampling_rate = max(dbs_data->min_sampling_rate,
385 latency * LATENCY_MULTIPLIER)); 485 LATENCY_MULTIPLIER * latency);
386 486
387 if (!have_governor_per_policy()) 487 if (!have_governor_per_policy())
388 cdata->gdbs_data = dbs_data; 488 gov->gdbs_data = dbs_data;
389 489
390 policy->governor_data = dbs_data; 490 policy->governor_data = policy_dbs;
391 491
392 ret = sysfs_create_group(get_governor_parent_kobj(policy), 492 policy_dbs->dbs_data = dbs_data;
393 get_sysfs_attr(dbs_data)); 493 dbs_data->usage_count = 1;
394 if (ret) 494 list_add(&policy_dbs->list, &dbs_data->policy_dbs_list);
395 goto reset_gdbs_data;
396 495
397 return 0; 496 gov->kobj_type.sysfs_ops = &governor_sysfs_ops;
497 ret = kobject_init_and_add(&dbs_data->kobj, &gov->kobj_type,
498 get_governor_parent_kobj(policy),
499 "%s", gov->gov.name);
500 if (!ret)
501 goto out;
502
503 /* Failure, so roll back. */
504 pr_err("cpufreq: Governor initialization failed (dbs_data kobject init error %d)\n", ret);
398 505
399reset_gdbs_data:
400 policy->governor_data = NULL; 506 policy->governor_data = NULL;
401 507
402 if (!have_governor_per_policy()) 508 if (!have_governor_per_policy())
403 cdata->gdbs_data = NULL; 509 gov->gdbs_data = NULL;
404 cdata->exit(dbs_data, !policy->governor->initialized); 510 gov->exit(dbs_data, !policy->governor->initialized);
405free_common_dbs_info:
406 free_common_dbs_info(policy, cdata);
407free_dbs_data:
408 kfree(dbs_data); 511 kfree(dbs_data);
512
513free_policy_dbs_info:
514 free_policy_dbs_info(policy_dbs, gov);
515
516out:
517 mutex_unlock(&gov_dbs_data_mutex);
409 return ret; 518 return ret;
410} 519}
411 520
412static int cpufreq_governor_exit(struct cpufreq_policy *policy, 521static int cpufreq_governor_exit(struct cpufreq_policy *policy)
413 struct dbs_data *dbs_data)
414{ 522{
415 struct common_dbs_data *cdata = dbs_data->cdata; 523 struct dbs_governor *gov = dbs_governor_of(policy);
416 struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(policy->cpu); 524 struct policy_dbs_info *policy_dbs = policy->governor_data;
525 struct dbs_data *dbs_data = policy_dbs->dbs_data;
526 int count;
417 527
418 /* State should be equivalent to INIT */ 528 /* Protect gov->gdbs_data against concurrent updates. */
419 if (!cdbs->shared || cdbs->shared->policy) 529 mutex_lock(&gov_dbs_data_mutex);
420 return -EBUSY; 530
531 mutex_lock(&dbs_data->mutex);
532 list_del(&policy_dbs->list);
533 count = --dbs_data->usage_count;
534 mutex_unlock(&dbs_data->mutex);
421 535
422 if (!--dbs_data->usage_count) { 536 if (!count) {
423 sysfs_remove_group(get_governor_parent_kobj(policy), 537 kobject_put(&dbs_data->kobj);
424 get_sysfs_attr(dbs_data));
425 538
426 policy->governor_data = NULL; 539 policy->governor_data = NULL;
427 540
428 if (!have_governor_per_policy()) 541 if (!have_governor_per_policy())
429 cdata->gdbs_data = NULL; 542 gov->gdbs_data = NULL;
430 543
431 cdata->exit(dbs_data, policy->governor->initialized == 1); 544 gov->exit(dbs_data, policy->governor->initialized == 1);
545 mutex_destroy(&dbs_data->mutex);
432 kfree(dbs_data); 546 kfree(dbs_data);
433 } else { 547 } else {
434 policy->governor_data = NULL; 548 policy->governor_data = NULL;
435 } 549 }
436 550
437 free_common_dbs_info(policy, cdata); 551 free_policy_dbs_info(policy_dbs, gov);
552
553 mutex_unlock(&gov_dbs_data_mutex);
438 return 0; 554 return 0;
439} 555}
440 556
441static int cpufreq_governor_start(struct cpufreq_policy *policy, 557static int cpufreq_governor_start(struct cpufreq_policy *policy)
442 struct dbs_data *dbs_data)
443{ 558{
444 struct common_dbs_data *cdata = dbs_data->cdata; 559 struct dbs_governor *gov = dbs_governor_of(policy);
445 unsigned int sampling_rate, ignore_nice, j, cpu = policy->cpu; 560 struct policy_dbs_info *policy_dbs = policy->governor_data;
446 struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu); 561 struct dbs_data *dbs_data = policy_dbs->dbs_data;
447 struct cpu_common_dbs_info *shared = cdbs->shared; 562 unsigned int sampling_rate, ignore_nice, j;
448 int io_busy = 0; 563 unsigned int io_busy;
449 564
450 if (!policy->cur) 565 if (!policy->cur)
451 return -EINVAL; 566 return -EINVAL;
452 567
453 /* State should be equivalent to INIT */ 568 policy_dbs->is_shared = policy_is_shared(policy);
454 if (!shared || shared->policy) 569 policy_dbs->rate_mult = 1;
455 return -EBUSY;
456 570
457 if (cdata->governor == GOV_CONSERVATIVE) { 571 sampling_rate = dbs_data->sampling_rate;
458 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 572 ignore_nice = dbs_data->ignore_nice_load;
459 573 io_busy = dbs_data->io_is_busy;
460 sampling_rate = cs_tuners->sampling_rate;
461 ignore_nice = cs_tuners->ignore_nice_load;
462 } else {
463 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
464
465 sampling_rate = od_tuners->sampling_rate;
466 ignore_nice = od_tuners->ignore_nice_load;
467 io_busy = od_tuners->io_is_busy;
468 }
469
470 shared->policy = policy;
471 shared->time_stamp = ktime_get();
472 574
473 for_each_cpu(j, policy->cpus) { 575 for_each_cpu(j, policy->cpus) {
474 struct cpu_dbs_info *j_cdbs = cdata->get_cpu_cdbs(j); 576 struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
475 unsigned int prev_load; 577 unsigned int prev_load;
476 578
477 j_cdbs->prev_cpu_idle = 579 j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy);
478 get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy);
479 580
480 prev_load = (unsigned int)(j_cdbs->prev_cpu_wall - 581 prev_load = j_cdbs->prev_cpu_wall - j_cdbs->prev_cpu_idle;
481 j_cdbs->prev_cpu_idle); 582 j_cdbs->prev_load = 100 * prev_load / (unsigned int)j_cdbs->prev_cpu_wall;
482 j_cdbs->prev_load = 100 * prev_load /
483 (unsigned int)j_cdbs->prev_cpu_wall;
484 583
485 if (ignore_nice) 584 if (ignore_nice)
486 j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE]; 585 j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
487
488 __setup_timer(&j_cdbs->timer, dbs_timer_handler,
489 (unsigned long)j_cdbs,
490 TIMER_DEFERRABLE | TIMER_IRQSAFE);
491 } 586 }
492 587
493 if (cdata->governor == GOV_CONSERVATIVE) { 588 gov->start(policy);
494 struct cs_cpu_dbs_info_s *cs_dbs_info =
495 cdata->get_cpu_dbs_info_s(cpu);
496
497 cs_dbs_info->down_skip = 0;
498 cs_dbs_info->requested_freq = policy->cur;
499 } else {
500 struct od_ops *od_ops = cdata->gov_ops;
501 struct od_cpu_dbs_info_s *od_dbs_info = cdata->get_cpu_dbs_info_s(cpu);
502
503 od_dbs_info->rate_mult = 1;
504 od_dbs_info->sample_type = OD_NORMAL_SAMPLE;
505 od_ops->powersave_bias_init_cpu(cpu);
506 }
507 589
508 gov_add_timers(policy, delay_for_sampling_rate(sampling_rate)); 590 gov_set_update_util(policy_dbs, sampling_rate);
509 return 0; 591 return 0;
510} 592}
511 593
512static int cpufreq_governor_stop(struct cpufreq_policy *policy, 594static int cpufreq_governor_stop(struct cpufreq_policy *policy)
513 struct dbs_data *dbs_data)
514{ 595{
515 struct cpu_dbs_info *cdbs = dbs_data->cdata->get_cpu_cdbs(policy->cpu); 596 gov_cancel_work(policy);
516 struct cpu_common_dbs_info *shared = cdbs->shared;
517
518 /* State should be equivalent to START */
519 if (!shared || !shared->policy)
520 return -EBUSY;
521
522 gov_cancel_work(shared);
523 shared->policy = NULL;
524
525 return 0; 597 return 0;
526} 598}
527 599
528static int cpufreq_governor_limits(struct cpufreq_policy *policy, 600static int cpufreq_governor_limits(struct cpufreq_policy *policy)
529 struct dbs_data *dbs_data)
530{ 601{
531 struct common_dbs_data *cdata = dbs_data->cdata; 602 struct policy_dbs_info *policy_dbs = policy->governor_data;
532 unsigned int cpu = policy->cpu;
533 struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu);
534 603
535 /* State should be equivalent to START */ 604 mutex_lock(&policy_dbs->timer_mutex);
536 if (!cdbs->shared || !cdbs->shared->policy) 605
537 return -EBUSY; 606 if (policy->max < policy->cur)
607 __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H);
608 else if (policy->min > policy->cur)
609 __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L);
610
611 gov_update_sample_delay(policy_dbs, 0);
538 612
539 mutex_lock(&cdbs->shared->timer_mutex); 613 mutex_unlock(&policy_dbs->timer_mutex);
540 if (policy->max < cdbs->shared->policy->cur)
541 __cpufreq_driver_target(cdbs->shared->policy, policy->max,
542 CPUFREQ_RELATION_H);
543 else if (policy->min > cdbs->shared->policy->cur)
544 __cpufreq_driver_target(cdbs->shared->policy, policy->min,
545 CPUFREQ_RELATION_L);
546 dbs_check_cpu(dbs_data, cpu);
547 mutex_unlock(&cdbs->shared->timer_mutex);
548 614
549 return 0; 615 return 0;
550} 616}
551 617
552int cpufreq_governor_dbs(struct cpufreq_policy *policy, 618int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event)
553 struct common_dbs_data *cdata, unsigned int event)
554{ 619{
555 struct dbs_data *dbs_data; 620 if (event == CPUFREQ_GOV_POLICY_INIT) {
556 int ret; 621 return cpufreq_governor_init(policy);
557 622 } else if (policy->governor_data) {
558 /* Lock governor to block concurrent initialization of governor */ 623 switch (event) {
559 mutex_lock(&cdata->mutex); 624 case CPUFREQ_GOV_POLICY_EXIT:
560 625 return cpufreq_governor_exit(policy);
561 if (have_governor_per_policy()) 626 case CPUFREQ_GOV_START:
562 dbs_data = policy->governor_data; 627 return cpufreq_governor_start(policy);
563 else 628 case CPUFREQ_GOV_STOP:
564 dbs_data = cdata->gdbs_data; 629 return cpufreq_governor_stop(policy);
565 630 case CPUFREQ_GOV_LIMITS:
566 if (!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT)) { 631 return cpufreq_governor_limits(policy);
567 ret = -EINVAL; 632 }
568 goto unlock;
569 }
570
571 switch (event) {
572 case CPUFREQ_GOV_POLICY_INIT:
573 ret = cpufreq_governor_init(policy, dbs_data, cdata);
574 break;
575 case CPUFREQ_GOV_POLICY_EXIT:
576 ret = cpufreq_governor_exit(policy, dbs_data);
577 break;
578 case CPUFREQ_GOV_START:
579 ret = cpufreq_governor_start(policy, dbs_data);
580 break;
581 case CPUFREQ_GOV_STOP:
582 ret = cpufreq_governor_stop(policy, dbs_data);
583 break;
584 case CPUFREQ_GOV_LIMITS:
585 ret = cpufreq_governor_limits(policy, dbs_data);
586 break;
587 default:
588 ret = -EINVAL;
589 } 633 }
590 634 return -EINVAL;
591unlock:
592 mutex_unlock(&cdata->mutex);
593
594 return ret;
595} 635}
596EXPORT_SYMBOL_GPL(cpufreq_governor_dbs); 636EXPORT_SYMBOL_GPL(cpufreq_governor_dbs);
diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h
index 91e767a058a7..61ff82fe0613 100644
--- a/drivers/cpufreq/cpufreq_governor.h
+++ b/drivers/cpufreq/cpufreq_governor.h
@@ -18,6 +18,7 @@
18#define _CPUFREQ_GOVERNOR_H 18#define _CPUFREQ_GOVERNOR_H
19 19
20#include <linux/atomic.h> 20#include <linux/atomic.h>
21#include <linux/irq_work.h>
21#include <linux/cpufreq.h> 22#include <linux/cpufreq.h>
22#include <linux/kernel_stat.h> 23#include <linux/kernel_stat.h>
23#include <linux/module.h> 24#include <linux/module.h>
@@ -41,96 +42,68 @@
41enum {OD_NORMAL_SAMPLE, OD_SUB_SAMPLE}; 42enum {OD_NORMAL_SAMPLE, OD_SUB_SAMPLE};
42 43
43/* 44/*
44 * Macro for creating governors sysfs routines 45 * Abbreviations:
45 * 46 * dbs: used as a shortform for demand based switching It helps to keep variable
46 * - gov_sys: One governor instance per whole system 47 * names smaller, simpler
47 * - gov_pol: One governor instance per policy 48 * cdbs: common dbs
49 * od_*: On-demand governor
50 * cs_*: Conservative governor
48 */ 51 */
49 52
50/* Create attributes */ 53/* Governor demand based switching data (per-policy or global). */
51#define gov_sys_attr_ro(_name) \ 54struct dbs_data {
52static struct global_attr _name##_gov_sys = \ 55 int usage_count;
53__ATTR(_name, 0444, show_##_name##_gov_sys, NULL) 56 void *tuners;
54 57 unsigned int min_sampling_rate;
55#define gov_sys_attr_rw(_name) \ 58 unsigned int ignore_nice_load;
56static struct global_attr _name##_gov_sys = \ 59 unsigned int sampling_rate;
57__ATTR(_name, 0644, show_##_name##_gov_sys, store_##_name##_gov_sys) 60 unsigned int sampling_down_factor;
58 61 unsigned int up_threshold;
59#define gov_pol_attr_ro(_name) \ 62 unsigned int io_is_busy;
60static struct freq_attr _name##_gov_pol = \
61__ATTR(_name, 0444, show_##_name##_gov_pol, NULL)
62
63#define gov_pol_attr_rw(_name) \
64static struct freq_attr _name##_gov_pol = \
65__ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol)
66 63
67#define gov_sys_pol_attr_rw(_name) \ 64 struct kobject kobj;
68 gov_sys_attr_rw(_name); \ 65 struct list_head policy_dbs_list;
69 gov_pol_attr_rw(_name) 66 /*
67 * Protect concurrent updates to governor tunables from sysfs,
68 * policy_dbs_list and usage_count.
69 */
70 struct mutex mutex;
71};
70 72
71#define gov_sys_pol_attr_ro(_name) \ 73/* Governor's specific attributes */
72 gov_sys_attr_ro(_name); \ 74struct dbs_data;
73 gov_pol_attr_ro(_name) 75struct governor_attr {
76 struct attribute attr;
77 ssize_t (*show)(struct dbs_data *dbs_data, char *buf);
78 ssize_t (*store)(struct dbs_data *dbs_data, const char *buf,
79 size_t count);
80};
74 81
75/* Create show/store routines */ 82#define gov_show_one(_gov, file_name) \
76#define show_one(_gov, file_name) \ 83static ssize_t show_##file_name \
77static ssize_t show_##file_name##_gov_sys \ 84(struct dbs_data *dbs_data, char *buf) \
78(struct kobject *kobj, struct attribute *attr, char *buf) \
79{ \ 85{ \
80 struct _gov##_dbs_tuners *tuners = _gov##_dbs_cdata.gdbs_data->tuners; \
81 return sprintf(buf, "%u\n", tuners->file_name); \
82} \
83 \
84static ssize_t show_##file_name##_gov_pol \
85(struct cpufreq_policy *policy, char *buf) \
86{ \
87 struct dbs_data *dbs_data = policy->governor_data; \
88 struct _gov##_dbs_tuners *tuners = dbs_data->tuners; \ 86 struct _gov##_dbs_tuners *tuners = dbs_data->tuners; \
89 return sprintf(buf, "%u\n", tuners->file_name); \ 87 return sprintf(buf, "%u\n", tuners->file_name); \
90} 88}
91 89
92#define store_one(_gov, file_name) \ 90#define gov_show_one_common(file_name) \
93static ssize_t store_##file_name##_gov_sys \ 91static ssize_t show_##file_name \
94(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) \ 92(struct dbs_data *dbs_data, char *buf) \
95{ \
96 struct dbs_data *dbs_data = _gov##_dbs_cdata.gdbs_data; \
97 return store_##file_name(dbs_data, buf, count); \
98} \
99 \
100static ssize_t store_##file_name##_gov_pol \
101(struct cpufreq_policy *policy, const char *buf, size_t count) \
102{ \ 93{ \
103 struct dbs_data *dbs_data = policy->governor_data; \ 94 return sprintf(buf, "%u\n", dbs_data->file_name); \
104 return store_##file_name(dbs_data, buf, count); \
105} 95}
106 96
107#define show_store_one(_gov, file_name) \ 97#define gov_attr_ro(_name) \
108show_one(_gov, file_name); \ 98static struct governor_attr _name = \
109store_one(_gov, file_name) 99__ATTR(_name, 0444, show_##_name, NULL)
110 100
111/* create helper routines */ 101#define gov_attr_rw(_name) \
112#define define_get_cpu_dbs_routines(_dbs_info) \ 102static struct governor_attr _name = \
113static struct cpu_dbs_info *get_cpu_cdbs(int cpu) \ 103__ATTR(_name, 0644, show_##_name, store_##_name)
114{ \
115 return &per_cpu(_dbs_info, cpu).cdbs; \
116} \
117 \
118static void *get_cpu_dbs_info_s(int cpu) \
119{ \
120 return &per_cpu(_dbs_info, cpu); \
121}
122
123/*
124 * Abbreviations:
125 * dbs: used as a shortform for demand based switching It helps to keep variable
126 * names smaller, simpler
127 * cdbs: common dbs
128 * od_*: On-demand governor
129 * cs_*: Conservative governor
130 */
131 104
132/* Common to all CPUs of a policy */ 105/* Common to all CPUs of a policy */
133struct cpu_common_dbs_info { 106struct policy_dbs_info {
134 struct cpufreq_policy *policy; 107 struct cpufreq_policy *policy;
135 /* 108 /*
136 * Per policy mutex that serializes load evaluation from limit-change 109 * Per policy mutex that serializes load evaluation from limit-change
@@ -138,11 +111,27 @@ struct cpu_common_dbs_info {
138 */ 111 */
139 struct mutex timer_mutex; 112 struct mutex timer_mutex;
140 113
141 ktime_t time_stamp; 114 u64 last_sample_time;
142 atomic_t skip_work; 115 s64 sample_delay_ns;
116 atomic_t work_count;
117 struct irq_work irq_work;
143 struct work_struct work; 118 struct work_struct work;
119 /* dbs_data may be shared between multiple policy objects */
120 struct dbs_data *dbs_data;
121 struct list_head list;
122 /* Multiplier for increasing sample delay temporarily. */
123 unsigned int rate_mult;
124 /* Status indicators */
125 bool is_shared; /* This object is used by multiple CPUs */
126 bool work_in_progress; /* Work is being queued up or in progress */
144}; 127};
145 128
129static inline void gov_update_sample_delay(struct policy_dbs_info *policy_dbs,
130 unsigned int delay_us)
131{
132 policy_dbs->sample_delay_ns = delay_us * NSEC_PER_USEC;
133}
134
146/* Per cpu structures */ 135/* Per cpu structures */
147struct cpu_dbs_info { 136struct cpu_dbs_info {
148 u64 prev_cpu_idle; 137 u64 prev_cpu_idle;
@@ -155,54 +144,14 @@ struct cpu_dbs_info {
155 * wake-up from idle. 144 * wake-up from idle.
156 */ 145 */
157 unsigned int prev_load; 146 unsigned int prev_load;
158 struct timer_list timer; 147 struct update_util_data update_util;
159 struct cpu_common_dbs_info *shared; 148 struct policy_dbs_info *policy_dbs;
160};
161
162struct od_cpu_dbs_info_s {
163 struct cpu_dbs_info cdbs;
164 struct cpufreq_frequency_table *freq_table;
165 unsigned int freq_lo;
166 unsigned int freq_lo_jiffies;
167 unsigned int freq_hi_jiffies;
168 unsigned int rate_mult;
169 unsigned int sample_type:1;
170};
171
172struct cs_cpu_dbs_info_s {
173 struct cpu_dbs_info cdbs;
174 unsigned int down_skip;
175 unsigned int requested_freq;
176};
177
178/* Per policy Governors sysfs tunables */
179struct od_dbs_tuners {
180 unsigned int ignore_nice_load;
181 unsigned int sampling_rate;
182 unsigned int sampling_down_factor;
183 unsigned int up_threshold;
184 unsigned int powersave_bias;
185 unsigned int io_is_busy;
186};
187
188struct cs_dbs_tuners {
189 unsigned int ignore_nice_load;
190 unsigned int sampling_rate;
191 unsigned int sampling_down_factor;
192 unsigned int up_threshold;
193 unsigned int down_threshold;
194 unsigned int freq_step;
195}; 149};
196 150
197/* Common Governor data across policies */ 151/* Common Governor data across policies */
198struct dbs_data; 152struct dbs_governor {
199struct common_dbs_data { 153 struct cpufreq_governor gov;
200 /* Common across governors */ 154 struct kobj_type kobj_type;
201 #define GOV_ONDEMAND 0
202 #define GOV_CONSERVATIVE 1
203 int governor;
204 struct attribute_group *attr_group_gov_sys; /* one governor - system */
205 struct attribute_group *attr_group_gov_pol; /* one governor - policy */
206 155
207 /* 156 /*
208 * Common data for platforms that don't set 157 * Common data for platforms that don't set
@@ -210,74 +159,32 @@ struct common_dbs_data {
210 */ 159 */
211 struct dbs_data *gdbs_data; 160 struct dbs_data *gdbs_data;
212 161
213 struct cpu_dbs_info *(*get_cpu_cdbs)(int cpu); 162 unsigned int (*gov_dbs_timer)(struct cpufreq_policy *policy);
214 void *(*get_cpu_dbs_info_s)(int cpu); 163 struct policy_dbs_info *(*alloc)(void);
215 unsigned int (*gov_dbs_timer)(struct cpufreq_policy *policy, 164 void (*free)(struct policy_dbs_info *policy_dbs);
216 bool modify_all);
217 void (*gov_check_cpu)(int cpu, unsigned int load);
218 int (*init)(struct dbs_data *dbs_data, bool notify); 165 int (*init)(struct dbs_data *dbs_data, bool notify);
219 void (*exit)(struct dbs_data *dbs_data, bool notify); 166 void (*exit)(struct dbs_data *dbs_data, bool notify);
220 167 void (*start)(struct cpufreq_policy *policy);
221 /* Governor specific ops, see below */
222 void *gov_ops;
223
224 /*
225 * Protects governor's data (struct dbs_data and struct common_dbs_data)
226 */
227 struct mutex mutex;
228}; 168};
229 169
230/* Governor Per policy data */ 170static inline struct dbs_governor *dbs_governor_of(struct cpufreq_policy *policy)
231struct dbs_data { 171{
232 struct common_dbs_data *cdata; 172 return container_of(policy->governor, struct dbs_governor, gov);
233 unsigned int min_sampling_rate; 173}
234 int usage_count;
235 void *tuners;
236};
237 174
238/* Governor specific ops, will be passed to dbs_data->gov_ops */ 175/* Governor specific operations */
239struct od_ops { 176struct od_ops {
240 void (*powersave_bias_init_cpu)(int cpu);
241 unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy, 177 unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy,
242 unsigned int freq_next, unsigned int relation); 178 unsigned int freq_next, unsigned int relation);
243 void (*freq_increase)(struct cpufreq_policy *policy, unsigned int freq);
244}; 179};
245 180
246static inline int delay_for_sampling_rate(unsigned int sampling_rate) 181unsigned int dbs_update(struct cpufreq_policy *policy);
247{ 182int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event);
248 int delay = usecs_to_jiffies(sampling_rate);
249
250 /* We want all CPUs to do sampling nearly on same jiffy */
251 if (num_online_cpus() > 1)
252 delay -= jiffies % delay;
253
254 return delay;
255}
256
257#define declare_show_sampling_rate_min(_gov) \
258static ssize_t show_sampling_rate_min_gov_sys \
259(struct kobject *kobj, struct attribute *attr, char *buf) \
260{ \
261 struct dbs_data *dbs_data = _gov##_dbs_cdata.gdbs_data; \
262 return sprintf(buf, "%u\n", dbs_data->min_sampling_rate); \
263} \
264 \
265static ssize_t show_sampling_rate_min_gov_pol \
266(struct cpufreq_policy *policy, char *buf) \
267{ \
268 struct dbs_data *dbs_data = policy->governor_data; \
269 return sprintf(buf, "%u\n", dbs_data->min_sampling_rate); \
270}
271
272extern struct mutex cpufreq_governor_lock;
273
274void gov_add_timers(struct cpufreq_policy *policy, unsigned int delay);
275void gov_cancel_work(struct cpu_common_dbs_info *shared);
276void dbs_check_cpu(struct dbs_data *dbs_data, int cpu);
277int cpufreq_governor_dbs(struct cpufreq_policy *policy,
278 struct common_dbs_data *cdata, unsigned int event);
279void od_register_powersave_bias_handler(unsigned int (*f) 183void od_register_powersave_bias_handler(unsigned int (*f)
280 (struct cpufreq_policy *, unsigned int, unsigned int), 184 (struct cpufreq_policy *, unsigned int, unsigned int),
281 unsigned int powersave_bias); 185 unsigned int powersave_bias);
282void od_unregister_powersave_bias_handler(void); 186void od_unregister_powersave_bias_handler(void);
187ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf,
188 size_t count);
189void gov_update_cpu_data(struct dbs_data *dbs_data);
283#endif /* _CPUFREQ_GOVERNOR_H */ 190#endif /* _CPUFREQ_GOVERNOR_H */
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
index 929e193ac1c1..acd80272ded6 100644
--- a/drivers/cpufreq/cpufreq_ondemand.c
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -16,7 +16,8 @@
16#include <linux/percpu-defs.h> 16#include <linux/percpu-defs.h>
17#include <linux/slab.h> 17#include <linux/slab.h>
18#include <linux/tick.h> 18#include <linux/tick.h>
19#include "cpufreq_governor.h" 19
20#include "cpufreq_ondemand.h"
20 21
21/* On-demand governor macros */ 22/* On-demand governor macros */
22#define DEF_FREQUENCY_UP_THRESHOLD (80) 23#define DEF_FREQUENCY_UP_THRESHOLD (80)
@@ -27,22 +28,10 @@
27#define MIN_FREQUENCY_UP_THRESHOLD (11) 28#define MIN_FREQUENCY_UP_THRESHOLD (11)
28#define MAX_FREQUENCY_UP_THRESHOLD (100) 29#define MAX_FREQUENCY_UP_THRESHOLD (100)
29 30
30static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info);
31
32static struct od_ops od_ops; 31static struct od_ops od_ops;
33 32
34static struct cpufreq_governor cpufreq_gov_ondemand;
35
36static unsigned int default_powersave_bias; 33static unsigned int default_powersave_bias;
37 34
38static void ondemand_powersave_bias_init_cpu(int cpu)
39{
40 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
41
42 dbs_info->freq_table = cpufreq_frequency_get_table(cpu);
43 dbs_info->freq_lo = 0;
44}
45
46/* 35/*
47 * Not all CPUs want IO time to be accounted as busy; this depends on how 36 * Not all CPUs want IO time to be accounted as busy; this depends on how
48 * efficient idling at a higher frequency/voltage is. 37 * efficient idling at a higher frequency/voltage is.
@@ -68,8 +57,8 @@ static int should_io_be_busy(void)
68 57
69/* 58/*
70 * Find right freq to be set now with powersave_bias on. 59 * Find right freq to be set now with powersave_bias on.
71 * Returns the freq_hi to be used right now and will set freq_hi_jiffies, 60 * Returns the freq_hi to be used right now and will set freq_hi_delay_us,
72 * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs. 61 * freq_lo, and freq_lo_delay_us in percpu area for averaging freqs.
73 */ 62 */
74static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy, 63static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy,
75 unsigned int freq_next, unsigned int relation) 64 unsigned int freq_next, unsigned int relation)
@@ -77,15 +66,15 @@ static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy,
77 unsigned int freq_req, freq_reduc, freq_avg; 66 unsigned int freq_req, freq_reduc, freq_avg;
78 unsigned int freq_hi, freq_lo; 67 unsigned int freq_hi, freq_lo;
79 unsigned int index = 0; 68 unsigned int index = 0;
80 unsigned int jiffies_total, jiffies_hi, jiffies_lo; 69 unsigned int delay_hi_us;
81 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, 70 struct policy_dbs_info *policy_dbs = policy->governor_data;
82 policy->cpu); 71 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
83 struct dbs_data *dbs_data = policy->governor_data; 72 struct dbs_data *dbs_data = policy_dbs->dbs_data;
84 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 73 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
85 74
86 if (!dbs_info->freq_table) { 75 if (!dbs_info->freq_table) {
87 dbs_info->freq_lo = 0; 76 dbs_info->freq_lo = 0;
88 dbs_info->freq_lo_jiffies = 0; 77 dbs_info->freq_lo_delay_us = 0;
89 return freq_next; 78 return freq_next;
90 } 79 }
91 80
@@ -108,31 +97,30 @@ static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy,
108 /* Find out how long we have to be in hi and lo freqs */ 97 /* Find out how long we have to be in hi and lo freqs */
109 if (freq_hi == freq_lo) { 98 if (freq_hi == freq_lo) {
110 dbs_info->freq_lo = 0; 99 dbs_info->freq_lo = 0;
111 dbs_info->freq_lo_jiffies = 0; 100 dbs_info->freq_lo_delay_us = 0;
112 return freq_lo; 101 return freq_lo;
113 } 102 }
114 jiffies_total = usecs_to_jiffies(od_tuners->sampling_rate); 103 delay_hi_us = (freq_avg - freq_lo) * dbs_data->sampling_rate;
115 jiffies_hi = (freq_avg - freq_lo) * jiffies_total; 104 delay_hi_us += (freq_hi - freq_lo) / 2;
116 jiffies_hi += ((freq_hi - freq_lo) / 2); 105 delay_hi_us /= freq_hi - freq_lo;
117 jiffies_hi /= (freq_hi - freq_lo); 106 dbs_info->freq_hi_delay_us = delay_hi_us;
118 jiffies_lo = jiffies_total - jiffies_hi;
119 dbs_info->freq_lo = freq_lo; 107 dbs_info->freq_lo = freq_lo;
120 dbs_info->freq_lo_jiffies = jiffies_lo; 108 dbs_info->freq_lo_delay_us = dbs_data->sampling_rate - delay_hi_us;
121 dbs_info->freq_hi_jiffies = jiffies_hi;
122 return freq_hi; 109 return freq_hi;
123} 110}
124 111
125static void ondemand_powersave_bias_init(void) 112static void ondemand_powersave_bias_init(struct cpufreq_policy *policy)
126{ 113{
127 int i; 114 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
128 for_each_online_cpu(i) { 115
129 ondemand_powersave_bias_init_cpu(i); 116 dbs_info->freq_table = cpufreq_frequency_get_table(policy->cpu);
130 } 117 dbs_info->freq_lo = 0;
131} 118}
132 119
133static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq) 120static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq)
134{ 121{
135 struct dbs_data *dbs_data = policy->governor_data; 122 struct policy_dbs_info *policy_dbs = policy->governor_data;
123 struct dbs_data *dbs_data = policy_dbs->dbs_data;
136 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 124 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
137 125
138 if (od_tuners->powersave_bias) 126 if (od_tuners->powersave_bias)
@@ -150,21 +138,21 @@ static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq)
150 * (default), then we try to increase frequency. Else, we adjust the frequency 138 * (default), then we try to increase frequency. Else, we adjust the frequency
151 * proportional to load. 139 * proportional to load.
152 */ 140 */
153static void od_check_cpu(int cpu, unsigned int load) 141static void od_update(struct cpufreq_policy *policy)
154{ 142{
155 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); 143 struct policy_dbs_info *policy_dbs = policy->governor_data;
156 struct cpufreq_policy *policy = dbs_info->cdbs.shared->policy; 144 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
157 struct dbs_data *dbs_data = policy->governor_data; 145 struct dbs_data *dbs_data = policy_dbs->dbs_data;
158 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 146 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
147 unsigned int load = dbs_update(policy);
159 148
160 dbs_info->freq_lo = 0; 149 dbs_info->freq_lo = 0;
161 150
162 /* Check for frequency increase */ 151 /* Check for frequency increase */
163 if (load > od_tuners->up_threshold) { 152 if (load > dbs_data->up_threshold) {
164 /* If switching to max speed, apply sampling_down_factor */ 153 /* If switching to max speed, apply sampling_down_factor */
165 if (policy->cur < policy->max) 154 if (policy->cur < policy->max)
166 dbs_info->rate_mult = 155 policy_dbs->rate_mult = dbs_data->sampling_down_factor;
167 od_tuners->sampling_down_factor;
168 dbs_freq_increase(policy, policy->max); 156 dbs_freq_increase(policy, policy->max);
169 } else { 157 } else {
170 /* Calculate the next frequency proportional to load */ 158 /* Calculate the next frequency proportional to load */
@@ -175,177 +163,70 @@ static void od_check_cpu(int cpu, unsigned int load)
175 freq_next = min_f + load * (max_f - min_f) / 100; 163 freq_next = min_f + load * (max_f - min_f) / 100;
176 164
177 /* No longer fully busy, reset rate_mult */ 165 /* No longer fully busy, reset rate_mult */
178 dbs_info->rate_mult = 1; 166 policy_dbs->rate_mult = 1;
179 167
180 if (!od_tuners->powersave_bias) { 168 if (od_tuners->powersave_bias)
181 __cpufreq_driver_target(policy, freq_next, 169 freq_next = od_ops.powersave_bias_target(policy,
182 CPUFREQ_RELATION_C); 170 freq_next,
183 return; 171 CPUFREQ_RELATION_L);
184 }
185 172
186 freq_next = od_ops.powersave_bias_target(policy, freq_next,
187 CPUFREQ_RELATION_L);
188 __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_C); 173 __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_C);
189 } 174 }
190} 175}
191 176
192static unsigned int od_dbs_timer(struct cpufreq_policy *policy, bool modify_all) 177static unsigned int od_dbs_timer(struct cpufreq_policy *policy)
193{ 178{
194 struct dbs_data *dbs_data = policy->governor_data; 179 struct policy_dbs_info *policy_dbs = policy->governor_data;
195 unsigned int cpu = policy->cpu; 180 struct dbs_data *dbs_data = policy_dbs->dbs_data;
196 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, 181 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
197 cpu); 182 int sample_type = dbs_info->sample_type;
198 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
199 int delay = 0, sample_type = dbs_info->sample_type;
200
201 if (!modify_all)
202 goto max_delay;
203 183
204 /* Common NORMAL_SAMPLE setup */ 184 /* Common NORMAL_SAMPLE setup */
205 dbs_info->sample_type = OD_NORMAL_SAMPLE; 185 dbs_info->sample_type = OD_NORMAL_SAMPLE;
206 if (sample_type == OD_SUB_SAMPLE) { 186 /*
207 delay = dbs_info->freq_lo_jiffies; 187 * OD_SUB_SAMPLE doesn't make sense if sample_delay_ns is 0, so ignore
188 * it then.
189 */
190 if (sample_type == OD_SUB_SAMPLE && policy_dbs->sample_delay_ns > 0) {
208 __cpufreq_driver_target(policy, dbs_info->freq_lo, 191 __cpufreq_driver_target(policy, dbs_info->freq_lo,
209 CPUFREQ_RELATION_H); 192 CPUFREQ_RELATION_H);
210 } else { 193 return dbs_info->freq_lo_delay_us;
211 dbs_check_cpu(dbs_data, cpu);
212 if (dbs_info->freq_lo) {
213 /* Setup timer for SUB_SAMPLE */
214 dbs_info->sample_type = OD_SUB_SAMPLE;
215 delay = dbs_info->freq_hi_jiffies;
216 }
217 } 194 }
218 195
219max_delay: 196 od_update(policy);
220 if (!delay)
221 delay = delay_for_sampling_rate(od_tuners->sampling_rate
222 * dbs_info->rate_mult);
223
224 return delay;
225}
226
227/************************** sysfs interface ************************/
228static struct common_dbs_data od_dbs_cdata;
229 197
230/** 198 if (dbs_info->freq_lo) {
231 * update_sampling_rate - update sampling rate effective immediately if needed. 199 /* Setup timer for SUB_SAMPLE */
232 * @new_rate: new sampling rate 200 dbs_info->sample_type = OD_SUB_SAMPLE;
233 * 201 return dbs_info->freq_hi_delay_us;
234 * If new rate is smaller than the old, simply updating
235 * dbs_tuners_int.sampling_rate might not be appropriate. For example, if the
236 * original sampling_rate was 1 second and the requested new sampling rate is 10
237 * ms because the user needs immediate reaction from ondemand governor, but not
238 * sure if higher frequency will be required or not, then, the governor may
239 * change the sampling rate too late; up to 1 second later. Thus, if we are
240 * reducing the sampling rate, we need to make the new value effective
241 * immediately.
242 */
243static void update_sampling_rate(struct dbs_data *dbs_data,
244 unsigned int new_rate)
245{
246 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
247 struct cpumask cpumask;
248 int cpu;
249
250 od_tuners->sampling_rate = new_rate = max(new_rate,
251 dbs_data->min_sampling_rate);
252
253 /*
254 * Lock governor so that governor start/stop can't execute in parallel.
255 */
256 mutex_lock(&od_dbs_cdata.mutex);
257
258 cpumask_copy(&cpumask, cpu_online_mask);
259
260 for_each_cpu(cpu, &cpumask) {
261 struct cpufreq_policy *policy;
262 struct od_cpu_dbs_info_s *dbs_info;
263 struct cpu_dbs_info *cdbs;
264 struct cpu_common_dbs_info *shared;
265 unsigned long next_sampling, appointed_at;
266
267 dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
268 cdbs = &dbs_info->cdbs;
269 shared = cdbs->shared;
270
271 /*
272 * A valid shared and shared->policy means governor hasn't
273 * stopped or exited yet.
274 */
275 if (!shared || !shared->policy)
276 continue;
277
278 policy = shared->policy;
279
280 /* clear all CPUs of this policy */
281 cpumask_andnot(&cpumask, &cpumask, policy->cpus);
282
283 /*
284 * Update sampling rate for CPUs whose policy is governed by
285 * dbs_data. In case of governor_per_policy, only a single
286 * policy will be governed by dbs_data, otherwise there can be
287 * multiple policies that are governed by the same dbs_data.
288 */
289 if (dbs_data != policy->governor_data)
290 continue;
291
292 /*
293 * Checking this for any CPU should be fine, timers for all of
294 * them are scheduled together.
295 */
296 next_sampling = jiffies + usecs_to_jiffies(new_rate);
297 appointed_at = dbs_info->cdbs.timer.expires;
298
299 if (time_before(next_sampling, appointed_at)) {
300 gov_cancel_work(shared);
301 gov_add_timers(policy, usecs_to_jiffies(new_rate));
302
303 }
304 } 202 }
305 203
306 mutex_unlock(&od_dbs_cdata.mutex); 204 return dbs_data->sampling_rate * policy_dbs->rate_mult;
307} 205}
308 206
309static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf, 207/************************** sysfs interface ************************/
310 size_t count) 208static struct dbs_governor od_dbs_gov;
311{
312 unsigned int input;
313 int ret;
314 ret = sscanf(buf, "%u", &input);
315 if (ret != 1)
316 return -EINVAL;
317
318 update_sampling_rate(dbs_data, input);
319 return count;
320}
321 209
322static ssize_t store_io_is_busy(struct dbs_data *dbs_data, const char *buf, 210static ssize_t store_io_is_busy(struct dbs_data *dbs_data, const char *buf,
323 size_t count) 211 size_t count)
324{ 212{
325 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
326 unsigned int input; 213 unsigned int input;
327 int ret; 214 int ret;
328 unsigned int j;
329 215
330 ret = sscanf(buf, "%u", &input); 216 ret = sscanf(buf, "%u", &input);
331 if (ret != 1) 217 if (ret != 1)
332 return -EINVAL; 218 return -EINVAL;
333 od_tuners->io_is_busy = !!input; 219 dbs_data->io_is_busy = !!input;
334 220
335 /* we need to re-evaluate prev_cpu_idle */ 221 /* we need to re-evaluate prev_cpu_idle */
336 for_each_online_cpu(j) { 222 gov_update_cpu_data(dbs_data);
337 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, 223
338 j);
339 dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j,
340 &dbs_info->cdbs.prev_cpu_wall, od_tuners->io_is_busy);
341 }
342 return count; 224 return count;
343} 225}
344 226
345static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf, 227static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf,
346 size_t count) 228 size_t count)
347{ 229{
348 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
349 unsigned int input; 230 unsigned int input;
350 int ret; 231 int ret;
351 ret = sscanf(buf, "%u", &input); 232 ret = sscanf(buf, "%u", &input);
@@ -355,40 +236,43 @@ static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf,
355 return -EINVAL; 236 return -EINVAL;
356 } 237 }
357 238
358 od_tuners->up_threshold = input; 239 dbs_data->up_threshold = input;
359 return count; 240 return count;
360} 241}
361 242
362static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data, 243static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data,
363 const char *buf, size_t count) 244 const char *buf, size_t count)
364{ 245{
365 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 246 struct policy_dbs_info *policy_dbs;
366 unsigned int input, j; 247 unsigned int input;
367 int ret; 248 int ret;
368 ret = sscanf(buf, "%u", &input); 249 ret = sscanf(buf, "%u", &input);
369 250
370 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) 251 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
371 return -EINVAL; 252 return -EINVAL;
372 od_tuners->sampling_down_factor = input; 253
254 dbs_data->sampling_down_factor = input;
373 255
374 /* Reset down sampling multiplier in case it was active */ 256 /* Reset down sampling multiplier in case it was active */
375 for_each_online_cpu(j) { 257 list_for_each_entry(policy_dbs, &dbs_data->policy_dbs_list, list) {
376 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, 258 /*
377 j); 259 * Doing this without locking might lead to using different
378 dbs_info->rate_mult = 1; 260 * rate_mult values in od_update() and od_dbs_timer().
261 */
262 mutex_lock(&policy_dbs->timer_mutex);
263 policy_dbs->rate_mult = 1;
264 mutex_unlock(&policy_dbs->timer_mutex);
379 } 265 }
266
380 return count; 267 return count;
381} 268}
382 269
383static ssize_t store_ignore_nice_load(struct dbs_data *dbs_data, 270static ssize_t store_ignore_nice_load(struct dbs_data *dbs_data,
384 const char *buf, size_t count) 271 const char *buf, size_t count)
385{ 272{
386 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
387 unsigned int input; 273 unsigned int input;
388 int ret; 274 int ret;
389 275
390 unsigned int j;
391
392 ret = sscanf(buf, "%u", &input); 276 ret = sscanf(buf, "%u", &input);
393 if (ret != 1) 277 if (ret != 1)
394 return -EINVAL; 278 return -EINVAL;
@@ -396,22 +280,14 @@ static ssize_t store_ignore_nice_load(struct dbs_data *dbs_data,
396 if (input > 1) 280 if (input > 1)
397 input = 1; 281 input = 1;
398 282
399 if (input == od_tuners->ignore_nice_load) { /* nothing to do */ 283 if (input == dbs_data->ignore_nice_load) { /* nothing to do */
400 return count; 284 return count;
401 } 285 }
402 od_tuners->ignore_nice_load = input; 286 dbs_data->ignore_nice_load = input;
403 287
404 /* we need to re-evaluate prev_cpu_idle */ 288 /* we need to re-evaluate prev_cpu_idle */
405 for_each_online_cpu(j) { 289 gov_update_cpu_data(dbs_data);
406 struct od_cpu_dbs_info_s *dbs_info;
407 dbs_info = &per_cpu(od_cpu_dbs_info, j);
408 dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j,
409 &dbs_info->cdbs.prev_cpu_wall, od_tuners->io_is_busy);
410 if (od_tuners->ignore_nice_load)
411 dbs_info->cdbs.prev_cpu_nice =
412 kcpustat_cpu(j).cpustat[CPUTIME_NICE];
413 290
414 }
415 return count; 291 return count;
416} 292}
417 293
@@ -419,6 +295,7 @@ static ssize_t store_powersave_bias(struct dbs_data *dbs_data, const char *buf,
419 size_t count) 295 size_t count)
420{ 296{
421 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 297 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
298 struct policy_dbs_info *policy_dbs;
422 unsigned int input; 299 unsigned int input;
423 int ret; 300 int ret;
424 ret = sscanf(buf, "%u", &input); 301 ret = sscanf(buf, "%u", &input);
@@ -430,59 +307,54 @@ static ssize_t store_powersave_bias(struct dbs_data *dbs_data, const char *buf,
430 input = 1000; 307 input = 1000;
431 308
432 od_tuners->powersave_bias = input; 309 od_tuners->powersave_bias = input;
433 ondemand_powersave_bias_init(); 310
311 list_for_each_entry(policy_dbs, &dbs_data->policy_dbs_list, list)
312 ondemand_powersave_bias_init(policy_dbs->policy);
313
434 return count; 314 return count;
435} 315}
436 316
437show_store_one(od, sampling_rate); 317gov_show_one_common(sampling_rate);
438show_store_one(od, io_is_busy); 318gov_show_one_common(up_threshold);
439show_store_one(od, up_threshold); 319gov_show_one_common(sampling_down_factor);
440show_store_one(od, sampling_down_factor); 320gov_show_one_common(ignore_nice_load);
441show_store_one(od, ignore_nice_load); 321gov_show_one_common(min_sampling_rate);
442show_store_one(od, powersave_bias); 322gov_show_one_common(io_is_busy);
443declare_show_sampling_rate_min(od); 323gov_show_one(od, powersave_bias);
444 324
445gov_sys_pol_attr_rw(sampling_rate); 325gov_attr_rw(sampling_rate);
446gov_sys_pol_attr_rw(io_is_busy); 326gov_attr_rw(io_is_busy);
447gov_sys_pol_attr_rw(up_threshold); 327gov_attr_rw(up_threshold);
448gov_sys_pol_attr_rw(sampling_down_factor); 328gov_attr_rw(sampling_down_factor);
449gov_sys_pol_attr_rw(ignore_nice_load); 329gov_attr_rw(ignore_nice_load);
450gov_sys_pol_attr_rw(powersave_bias); 330gov_attr_rw(powersave_bias);
451gov_sys_pol_attr_ro(sampling_rate_min); 331gov_attr_ro(min_sampling_rate);
452 332
453static struct attribute *dbs_attributes_gov_sys[] = { 333static struct attribute *od_attributes[] = {
454 &sampling_rate_min_gov_sys.attr, 334 &min_sampling_rate.attr,
455 &sampling_rate_gov_sys.attr, 335 &sampling_rate.attr,
456 &up_threshold_gov_sys.attr, 336 &up_threshold.attr,
457 &sampling_down_factor_gov_sys.attr, 337 &sampling_down_factor.attr,
458 &ignore_nice_load_gov_sys.attr, 338 &ignore_nice_load.attr,
459 &powersave_bias_gov_sys.attr, 339 &powersave_bias.attr,
460 &io_is_busy_gov_sys.attr, 340 &io_is_busy.attr,
461 NULL 341 NULL
462}; 342};
463 343
464static struct attribute_group od_attr_group_gov_sys = { 344/************************** sysfs end ************************/
465 .attrs = dbs_attributes_gov_sys,
466 .name = "ondemand",
467};
468 345
469static struct attribute *dbs_attributes_gov_pol[] = { 346static struct policy_dbs_info *od_alloc(void)
470 &sampling_rate_min_gov_pol.attr, 347{
471 &sampling_rate_gov_pol.attr, 348 struct od_policy_dbs_info *dbs_info;
472 &up_threshold_gov_pol.attr,
473 &sampling_down_factor_gov_pol.attr,
474 &ignore_nice_load_gov_pol.attr,
475 &powersave_bias_gov_pol.attr,
476 &io_is_busy_gov_pol.attr,
477 NULL
478};
479 349
480static struct attribute_group od_attr_group_gov_pol = { 350 dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL);
481 .attrs = dbs_attributes_gov_pol, 351 return dbs_info ? &dbs_info->policy_dbs : NULL;
482 .name = "ondemand", 352}
483};
484 353
485/************************** sysfs end ************************/ 354static void od_free(struct policy_dbs_info *policy_dbs)
355{
356 kfree(to_dbs_info(policy_dbs));
357}
486 358
487static int od_init(struct dbs_data *dbs_data, bool notify) 359static int od_init(struct dbs_data *dbs_data, bool notify)
488{ 360{
@@ -501,7 +373,7 @@ static int od_init(struct dbs_data *dbs_data, bool notify)
501 put_cpu(); 373 put_cpu();
502 if (idle_time != -1ULL) { 374 if (idle_time != -1ULL) {
503 /* Idle micro accounting is supported. Use finer thresholds */ 375 /* Idle micro accounting is supported. Use finer thresholds */
504 tuners->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; 376 dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
505 /* 377 /*
506 * In nohz/micro accounting case we set the minimum frequency 378 * In nohz/micro accounting case we set the minimum frequency
507 * not depending on HZ, but fixed (very low). The deferred 379 * not depending on HZ, but fixed (very low). The deferred
@@ -509,17 +381,17 @@ static int od_init(struct dbs_data *dbs_data, bool notify)
509 */ 381 */
510 dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; 382 dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
511 } else { 383 } else {
512 tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; 384 dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
513 385
514 /* For correct statistics, we need 10 ticks for each measure */ 386 /* For correct statistics, we need 10 ticks for each measure */
515 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * 387 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
516 jiffies_to_usecs(10); 388 jiffies_to_usecs(10);
517 } 389 }
518 390
519 tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; 391 dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
520 tuners->ignore_nice_load = 0; 392 dbs_data->ignore_nice_load = 0;
521 tuners->powersave_bias = default_powersave_bias; 393 tuners->powersave_bias = default_powersave_bias;
522 tuners->io_is_busy = should_io_be_busy(); 394 dbs_data->io_is_busy = should_io_be_busy();
523 395
524 dbs_data->tuners = tuners; 396 dbs_data->tuners = tuners;
525 return 0; 397 return 0;
@@ -530,46 +402,38 @@ static void od_exit(struct dbs_data *dbs_data, bool notify)
530 kfree(dbs_data->tuners); 402 kfree(dbs_data->tuners);
531} 403}
532 404
533define_get_cpu_dbs_routines(od_cpu_dbs_info); 405static void od_start(struct cpufreq_policy *policy)
406{
407 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
408
409 dbs_info->sample_type = OD_NORMAL_SAMPLE;
410 ondemand_powersave_bias_init(policy);
411}
534 412
535static struct od_ops od_ops = { 413static struct od_ops od_ops = {
536 .powersave_bias_init_cpu = ondemand_powersave_bias_init_cpu,
537 .powersave_bias_target = generic_powersave_bias_target, 414 .powersave_bias_target = generic_powersave_bias_target,
538 .freq_increase = dbs_freq_increase,
539}; 415};
540 416
541static struct common_dbs_data od_dbs_cdata = { 417static struct dbs_governor od_dbs_gov = {
542 .governor = GOV_ONDEMAND, 418 .gov = {
543 .attr_group_gov_sys = &od_attr_group_gov_sys, 419 .name = "ondemand",
544 .attr_group_gov_pol = &od_attr_group_gov_pol, 420 .governor = cpufreq_governor_dbs,
545 .get_cpu_cdbs = get_cpu_cdbs, 421 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
546 .get_cpu_dbs_info_s = get_cpu_dbs_info_s, 422 .owner = THIS_MODULE,
423 },
424 .kobj_type = { .default_attrs = od_attributes },
547 .gov_dbs_timer = od_dbs_timer, 425 .gov_dbs_timer = od_dbs_timer,
548 .gov_check_cpu = od_check_cpu, 426 .alloc = od_alloc,
549 .gov_ops = &od_ops, 427 .free = od_free,
550 .init = od_init, 428 .init = od_init,
551 .exit = od_exit, 429 .exit = od_exit,
552 .mutex = __MUTEX_INITIALIZER(od_dbs_cdata.mutex), 430 .start = od_start,
553}; 431};
554 432
555static int od_cpufreq_governor_dbs(struct cpufreq_policy *policy, 433#define CPU_FREQ_GOV_ONDEMAND (&od_dbs_gov.gov)
556 unsigned int event)
557{
558 return cpufreq_governor_dbs(policy, &od_dbs_cdata, event);
559}
560
561static struct cpufreq_governor cpufreq_gov_ondemand = {
562 .name = "ondemand",
563 .governor = od_cpufreq_governor_dbs,
564 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
565 .owner = THIS_MODULE,
566};
567 434
568static void od_set_powersave_bias(unsigned int powersave_bias) 435static void od_set_powersave_bias(unsigned int powersave_bias)
569{ 436{
570 struct cpufreq_policy *policy;
571 struct dbs_data *dbs_data;
572 struct od_dbs_tuners *od_tuners;
573 unsigned int cpu; 437 unsigned int cpu;
574 cpumask_t done; 438 cpumask_t done;
575 439
@@ -578,22 +442,25 @@ static void od_set_powersave_bias(unsigned int powersave_bias)
578 442
579 get_online_cpus(); 443 get_online_cpus();
580 for_each_online_cpu(cpu) { 444 for_each_online_cpu(cpu) {
581 struct cpu_common_dbs_info *shared; 445 struct cpufreq_policy *policy;
446 struct policy_dbs_info *policy_dbs;
447 struct dbs_data *dbs_data;
448 struct od_dbs_tuners *od_tuners;
582 449
583 if (cpumask_test_cpu(cpu, &done)) 450 if (cpumask_test_cpu(cpu, &done))
584 continue; 451 continue;
585 452
586 shared = per_cpu(od_cpu_dbs_info, cpu).cdbs.shared; 453 policy = cpufreq_cpu_get_raw(cpu);
587 if (!shared) 454 if (!policy || policy->governor != CPU_FREQ_GOV_ONDEMAND)
588 continue; 455 continue;
589 456
590 policy = shared->policy; 457 policy_dbs = policy->governor_data;
591 cpumask_or(&done, &done, policy->cpus); 458 if (!policy_dbs)
592
593 if (policy->governor != &cpufreq_gov_ondemand)
594 continue; 459 continue;
595 460
596 dbs_data = policy->governor_data; 461 cpumask_or(&done, &done, policy->cpus);
462
463 dbs_data = policy_dbs->dbs_data;
597 od_tuners = dbs_data->tuners; 464 od_tuners = dbs_data->tuners;
598 od_tuners->powersave_bias = default_powersave_bias; 465 od_tuners->powersave_bias = default_powersave_bias;
599 } 466 }
@@ -618,12 +485,12 @@ EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler);
618 485
619static int __init cpufreq_gov_dbs_init(void) 486static int __init cpufreq_gov_dbs_init(void)
620{ 487{
621 return cpufreq_register_governor(&cpufreq_gov_ondemand); 488 return cpufreq_register_governor(CPU_FREQ_GOV_ONDEMAND);
622} 489}
623 490
624static void __exit cpufreq_gov_dbs_exit(void) 491static void __exit cpufreq_gov_dbs_exit(void)
625{ 492{
626 cpufreq_unregister_governor(&cpufreq_gov_ondemand); 493 cpufreq_unregister_governor(CPU_FREQ_GOV_ONDEMAND);
627} 494}
628 495
629MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>"); 496MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
@@ -635,7 +502,7 @@ MODULE_LICENSE("GPL");
635#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND 502#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
636struct cpufreq_governor *cpufreq_default_governor(void) 503struct cpufreq_governor *cpufreq_default_governor(void)
637{ 504{
638 return &cpufreq_gov_ondemand; 505 return CPU_FREQ_GOV_ONDEMAND;
639} 506}
640 507
641fs_initcall(cpufreq_gov_dbs_init); 508fs_initcall(cpufreq_gov_dbs_init);
diff --git a/drivers/cpufreq/cpufreq_ondemand.h b/drivers/cpufreq/cpufreq_ondemand.h
new file mode 100644
index 000000000000..f0121db3cd9e
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_ondemand.h
@@ -0,0 +1,30 @@
1/*
2 * Header file for CPUFreq ondemand governor and related code.
3 *
4 * Copyright (C) 2016, Intel Corporation
5 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include "cpufreq_governor.h"
13
14struct od_policy_dbs_info {
15 struct policy_dbs_info policy_dbs;
16 struct cpufreq_frequency_table *freq_table;
17 unsigned int freq_lo;
18 unsigned int freq_lo_delay_us;
19 unsigned int freq_hi_delay_us;
20 unsigned int sample_type:1;
21};
22
23static inline struct od_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs)
24{
25 return container_of(policy_dbs, struct od_policy_dbs_info, policy_dbs);
26}
27
28struct od_dbs_tuners {
29 unsigned int powersave_bias;
30};
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 937667065d31..23bb798d0cd2 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -71,7 +71,7 @@ struct sample {
71 u64 mperf; 71 u64 mperf;
72 u64 tsc; 72 u64 tsc;
73 int freq; 73 int freq;
74 ktime_t time; 74 u64 time;
75}; 75};
76 76
77struct pstate_data { 77struct pstate_data {
@@ -103,13 +103,13 @@ struct _pid {
103struct cpudata { 103struct cpudata {
104 int cpu; 104 int cpu;
105 105
106 struct timer_list timer; 106 struct update_util_data update_util;
107 107
108 struct pstate_data pstate; 108 struct pstate_data pstate;
109 struct vid_data vid; 109 struct vid_data vid;
110 struct _pid pid; 110 struct _pid pid;
111 111
112 ktime_t last_sample_time; 112 u64 last_sample_time;
113 u64 prev_aperf; 113 u64 prev_aperf;
114 u64 prev_mperf; 114 u64 prev_mperf;
115 u64 prev_tsc; 115 u64 prev_tsc;
@@ -120,6 +120,7 @@ struct cpudata {
120static struct cpudata **all_cpu_data; 120static struct cpudata **all_cpu_data;
121struct pstate_adjust_policy { 121struct pstate_adjust_policy {
122 int sample_rate_ms; 122 int sample_rate_ms;
123 s64 sample_rate_ns;
123 int deadband; 124 int deadband;
124 int setpoint; 125 int setpoint;
125 int p_gain_pct; 126 int p_gain_pct;
@@ -718,7 +719,7 @@ static void core_set_pstate(struct cpudata *cpudata, int pstate)
718 if (limits->no_turbo && !limits->turbo_disabled) 719 if (limits->no_turbo && !limits->turbo_disabled)
719 val |= (u64)1 << 32; 720 val |= (u64)1 << 32;
720 721
721 wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val); 722 wrmsrl(MSR_IA32_PERF_CTL, val);
722} 723}
723 724
724static int knl_get_turbo_pstate(void) 725static int knl_get_turbo_pstate(void)
@@ -889,7 +890,7 @@ static inline void intel_pstate_calc_busy(struct cpudata *cpu)
889 sample->core_pct_busy = (int32_t)core_pct; 890 sample->core_pct_busy = (int32_t)core_pct;
890} 891}
891 892
892static inline void intel_pstate_sample(struct cpudata *cpu) 893static inline void intel_pstate_sample(struct cpudata *cpu, u64 time)
893{ 894{
894 u64 aperf, mperf; 895 u64 aperf, mperf;
895 unsigned long flags; 896 unsigned long flags;
@@ -906,7 +907,7 @@ static inline void intel_pstate_sample(struct cpudata *cpu)
906 local_irq_restore(flags); 907 local_irq_restore(flags);
907 908
908 cpu->last_sample_time = cpu->sample.time; 909 cpu->last_sample_time = cpu->sample.time;
909 cpu->sample.time = ktime_get(); 910 cpu->sample.time = time;
910 cpu->sample.aperf = aperf; 911 cpu->sample.aperf = aperf;
911 cpu->sample.mperf = mperf; 912 cpu->sample.mperf = mperf;
912 cpu->sample.tsc = tsc; 913 cpu->sample.tsc = tsc;
@@ -921,22 +922,6 @@ static inline void intel_pstate_sample(struct cpudata *cpu)
921 cpu->prev_tsc = tsc; 922 cpu->prev_tsc = tsc;
922} 923}
923 924
924static inline void intel_hwp_set_sample_time(struct cpudata *cpu)
925{
926 int delay;
927
928 delay = msecs_to_jiffies(50);
929 mod_timer_pinned(&cpu->timer, jiffies + delay);
930}
931
932static inline void intel_pstate_set_sample_time(struct cpudata *cpu)
933{
934 int delay;
935
936 delay = msecs_to_jiffies(pid_params.sample_rate_ms);
937 mod_timer_pinned(&cpu->timer, jiffies + delay);
938}
939
940static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu) 925static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
941{ 926{
942 struct sample *sample = &cpu->sample; 927 struct sample *sample = &cpu->sample;
@@ -976,8 +961,7 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
976static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu) 961static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
977{ 962{
978 int32_t core_busy, max_pstate, current_pstate, sample_ratio; 963 int32_t core_busy, max_pstate, current_pstate, sample_ratio;
979 s64 duration_us; 964 u64 duration_ns;
980 u32 sample_time;
981 965
982 /* 966 /*
983 * core_busy is the ratio of actual performance to max 967 * core_busy is the ratio of actual performance to max
@@ -996,18 +980,16 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
996 core_busy = mul_fp(core_busy, div_fp(max_pstate, current_pstate)); 980 core_busy = mul_fp(core_busy, div_fp(max_pstate, current_pstate));
997 981
998 /* 982 /*
999 * Since we have a deferred timer, it will not fire unless 983 * Since our utilization update callback will not run unless we are
1000 * we are in C0. So, determine if the actual elapsed time 984 * in C0, check if the actual elapsed time is significantly greater (3x)
1001 * is significantly greater (3x) than our sample interval. If it 985 * than our sample interval. If it is, then we were idle for a long
1002 * is, then we were idle for a long enough period of time 986 * enough period of time to adjust our busyness.
1003 * to adjust our busyness.
1004 */ 987 */
1005 sample_time = pid_params.sample_rate_ms * USEC_PER_MSEC; 988 duration_ns = cpu->sample.time - cpu->last_sample_time;
1006 duration_us = ktime_us_delta(cpu->sample.time, 989 if ((s64)duration_ns > pid_params.sample_rate_ns * 3
1007 cpu->last_sample_time); 990 && cpu->last_sample_time > 0) {
1008 if (duration_us > sample_time * 3) { 991 sample_ratio = div_fp(int_tofp(pid_params.sample_rate_ns),
1009 sample_ratio = div_fp(int_tofp(sample_time), 992 int_tofp(duration_ns));
1010 int_tofp(duration_us));
1011 core_busy = mul_fp(core_busy, sample_ratio); 993 core_busy = mul_fp(core_busy, sample_ratio);
1012 } 994 }
1013 995
@@ -1037,23 +1019,17 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
1037 sample->freq); 1019 sample->freq);
1038} 1020}
1039 1021
1040static void intel_hwp_timer_func(unsigned long __data) 1022static void intel_pstate_update_util(struct update_util_data *data, u64 time,
1041{ 1023 unsigned long util, unsigned long max)
1042 struct cpudata *cpu = (struct cpudata *) __data;
1043
1044 intel_pstate_sample(cpu);
1045 intel_hwp_set_sample_time(cpu);
1046}
1047
1048static void intel_pstate_timer_func(unsigned long __data)
1049{ 1024{
1050 struct cpudata *cpu = (struct cpudata *) __data; 1025 struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1051 1026 u64 delta_ns = time - cpu->sample.time;
1052 intel_pstate_sample(cpu);
1053 1027
1054 intel_pstate_adjust_busy_pstate(cpu); 1028 if ((s64)delta_ns >= pid_params.sample_rate_ns) {
1055 1029 intel_pstate_sample(cpu, time);
1056 intel_pstate_set_sample_time(cpu); 1030 if (!hwp_active)
1031 intel_pstate_adjust_busy_pstate(cpu);
1032 }
1057} 1033}
1058 1034
1059#define ICPU(model, policy) \ 1035#define ICPU(model, policy) \
@@ -1101,24 +1077,19 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
1101 1077
1102 cpu->cpu = cpunum; 1078 cpu->cpu = cpunum;
1103 1079
1104 if (hwp_active) 1080 if (hwp_active) {
1105 intel_pstate_hwp_enable(cpu); 1081 intel_pstate_hwp_enable(cpu);
1082 pid_params.sample_rate_ms = 50;
1083 pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC;
1084 }
1106 1085
1107 intel_pstate_get_cpu_pstates(cpu); 1086 intel_pstate_get_cpu_pstates(cpu);
1108 1087
1109 init_timer_deferrable(&cpu->timer);
1110 cpu->timer.data = (unsigned long)cpu;
1111 cpu->timer.expires = jiffies + HZ/100;
1112
1113 if (!hwp_active)
1114 cpu->timer.function = intel_pstate_timer_func;
1115 else
1116 cpu->timer.function = intel_hwp_timer_func;
1117
1118 intel_pstate_busy_pid_reset(cpu); 1088 intel_pstate_busy_pid_reset(cpu);
1119 intel_pstate_sample(cpu); 1089 intel_pstate_sample(cpu, 0);
1120 1090
1121 add_timer_on(&cpu->timer, cpunum); 1091 cpu->update_util.func = intel_pstate_update_util;
1092 cpufreq_set_update_util_data(cpunum, &cpu->update_util);
1122 1093
1123 pr_debug("intel_pstate: controlling: cpu %d\n", cpunum); 1094 pr_debug("intel_pstate: controlling: cpu %d\n", cpunum);
1124 1095
@@ -1202,7 +1173,9 @@ static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
1202 1173
1203 pr_debug("intel_pstate: CPU %d exiting\n", cpu_num); 1174 pr_debug("intel_pstate: CPU %d exiting\n", cpu_num);
1204 1175
1205 del_timer_sync(&all_cpu_data[cpu_num]->timer); 1176 cpufreq_set_update_util_data(cpu_num, NULL);
1177 synchronize_sched();
1178
1206 if (hwp_active) 1179 if (hwp_active)
1207 return; 1180 return;
1208 1181
@@ -1266,6 +1239,7 @@ static int intel_pstate_msrs_not_valid(void)
1266static void copy_pid_params(struct pstate_adjust_policy *policy) 1239static void copy_pid_params(struct pstate_adjust_policy *policy)
1267{ 1240{
1268 pid_params.sample_rate_ms = policy->sample_rate_ms; 1241 pid_params.sample_rate_ms = policy->sample_rate_ms;
1242 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
1269 pid_params.p_gain_pct = policy->p_gain_pct; 1243 pid_params.p_gain_pct = policy->p_gain_pct;
1270 pid_params.i_gain_pct = policy->i_gain_pct; 1244 pid_params.i_gain_pct = policy->i_gain_pct;
1271 pid_params.d_gain_pct = policy->d_gain_pct; 1245 pid_params.d_gain_pct = policy->d_gain_pct;
@@ -1467,7 +1441,8 @@ out:
1467 get_online_cpus(); 1441 get_online_cpus();
1468 for_each_online_cpu(cpu) { 1442 for_each_online_cpu(cpu) {
1469 if (all_cpu_data[cpu]) { 1443 if (all_cpu_data[cpu]) {
1470 del_timer_sync(&all_cpu_data[cpu]->timer); 1444 cpufreq_set_update_util_data(cpu, NULL);
1445 synchronize_sched();
1471 kfree(all_cpu_data[cpu]); 1446 kfree(all_cpu_data[cpu]);
1472 } 1447 }
1473 } 1448 }
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-01 19:28:05 -0400