aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/cpu-freq/core.txt7
-rw-r--r--Documentation/devicetree/bindings/clock/mt8173-cpu-dvfs.txt83
-rw-r--r--arch/powerpc/include/asm/opal-api.h12
-rw-r--r--drivers/acpi/processor_perflib.c6
-rw-r--r--drivers/cpufreq/Kconfig.arm7
-rw-r--r--drivers/cpufreq/Makefile1
-rw-r--r--drivers/cpufreq/acpi-cpufreq.c93
-rw-r--r--drivers/cpufreq/cpufreq.c351
-rw-r--r--drivers/cpufreq/cpufreq_conservative.c25
-rw-r--r--drivers/cpufreq/cpufreq_governor.c196
-rw-r--r--drivers/cpufreq/cpufreq_governor.h40
-rw-r--r--drivers/cpufreq/cpufreq_ondemand.c67
-rw-r--r--drivers/cpufreq/e_powersaver.c2
-rw-r--r--drivers/cpufreq/ia64-acpi-cpufreq.c20
-rw-r--r--drivers/cpufreq/integrator-cpufreq.c18
-rw-r--r--drivers/cpufreq/intel_pstate.c20
-rw-r--r--drivers/cpufreq/mt8173-cpufreq.c527
-rw-r--r--drivers/cpufreq/powernow-k7.c4
-rw-r--r--drivers/cpufreq/powernow-k8.c5
-rw-r--r--drivers/cpufreq/powernv-cpufreq.c199
-rw-r--r--drivers/cpufreq/ppc_cbe_cpufreq_pmi.c4
-rw-r--r--drivers/cpufreq/sfi-cpufreq.c4
-rw-r--r--drivers/cpufreq/speedstep-lib.c9
-rw-r--r--drivers/video/fbdev/pxafb.c1
-rw-r--r--drivers/video/fbdev/sa1100fb.c1
-rw-r--r--drivers/xen/xen-acpi-processor.c16
-rw-r--r--include/acpi/processor.h5
-rw-r--r--include/linux/cpufreq.h15
28 files changed, 1280 insertions, 458 deletions
diff --git a/Documentation/cpu-freq/core.txt b/Documentation/cpu-freq/core.txt
index 70933eadc308..ba78e7c2a069 100644
--- a/Documentation/cpu-freq/core.txt
+++ b/Documentation/cpu-freq/core.txt
@@ -55,16 +55,13 @@ transition notifiers.
55---------------------------- 55----------------------------
56 56
57These are notified when a new policy is intended to be set. Each 57These are notified when a new policy is intended to be set. Each
58CPUFreq policy notifier is called three times for a policy transition: 58CPUFreq policy notifier is called twice for a policy transition:
59 59
601.) During CPUFREQ_ADJUST all CPUFreq notifiers may change the limit if 601.) During CPUFREQ_ADJUST all CPUFreq notifiers may change the limit if
61 they see a need for this - may it be thermal considerations or 61 they see a need for this - may it be thermal considerations or
62 hardware limitations. 62 hardware limitations.
63 63
642.) During CPUFREQ_INCOMPATIBLE only changes may be done in order to avoid 642.) And during CPUFREQ_NOTIFY all notifiers are informed of the new policy
65 hardware failure.
66
673.) And during CPUFREQ_NOTIFY all notifiers are informed of the new policy
68 - if two hardware drivers failed to agree on a new policy before this 65 - if two hardware drivers failed to agree on a new policy before this
69 stage, the incompatible hardware shall be shut down, and the user 66 stage, the incompatible hardware shall be shut down, and the user
70 informed of this. 67 informed of this.
diff --git a/Documentation/devicetree/bindings/clock/mt8173-cpu-dvfs.txt b/Documentation/devicetree/bindings/clock/mt8173-cpu-dvfs.txt
new file mode 100644
index 000000000000..52b457c23eed
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/mt8173-cpu-dvfs.txt
@@ -0,0 +1,83 @@
1Device Tree Clock bindins for CPU DVFS of Mediatek MT8173 SoC
2
3Required properties:
4- clocks: A list of phandle + clock-specifier pairs for the clocks listed in clock names.
5- clock-names: Should contain the following:
6 "cpu" - The multiplexer for clock input of CPU cluster.
7 "intermediate" - A parent of "cpu" clock which is used as "intermediate" clock
8 source (usually MAINPLL) when the original CPU PLL is under
9 transition and not stable yet.
10 Please refer to Documentation/devicetree/bindings/clk/clock-bindings.txt for
11 generic clock consumer properties.
12- proc-supply: Regulator for Vproc of CPU cluster.
13
14Optional properties:
15- sram-supply: Regulator for Vsram of CPU cluster. When present, the cpufreq driver
16 needs to do "voltage tracking" to step by step scale up/down Vproc and
17 Vsram to fit SoC specific needs. When absent, the voltage scaling
18 flow is handled by hardware, hence no software "voltage tracking" is
19 needed.
20
21Example:
22--------
23 cpu0: cpu@0 {
24 device_type = "cpu";
25 compatible = "arm,cortex-a53";
26 reg = <0x000>;
27 enable-method = "psci";
28 cpu-idle-states = <&CPU_SLEEP_0>;
29 clocks = <&infracfg CLK_INFRA_CA53SEL>,
30 <&apmixedsys CLK_APMIXED_MAINPLL>;
31 clock-names = "cpu", "intermediate";
32 };
33
34 cpu1: cpu@1 {
35 device_type = "cpu";
36 compatible = "arm,cortex-a53";
37 reg = <0x001>;
38 enable-method = "psci";
39 cpu-idle-states = <&CPU_SLEEP_0>;
40 clocks = <&infracfg CLK_INFRA_CA53SEL>,
41 <&apmixedsys CLK_APMIXED_MAINPLL>;
42 clock-names = "cpu", "intermediate";
43 };
44
45 cpu2: cpu@100 {
46 device_type = "cpu";
47 compatible = "arm,cortex-a57";
48 reg = <0x100>;
49 enable-method = "psci";
50 cpu-idle-states = <&CPU_SLEEP_0>;
51 clocks = <&infracfg CLK_INFRA_CA57SEL>,
52 <&apmixedsys CLK_APMIXED_MAINPLL>;
53 clock-names = "cpu", "intermediate";
54 };
55
56 cpu3: cpu@101 {
57 device_type = "cpu";
58 compatible = "arm,cortex-a57";
59 reg = <0x101>;
60 enable-method = "psci";
61 cpu-idle-states = <&CPU_SLEEP_0>;
62 clocks = <&infracfg CLK_INFRA_CA57SEL>,
63 <&apmixedsys CLK_APMIXED_MAINPLL>;
64 clock-names = "cpu", "intermediate";
65 };
66
67 &cpu0 {
68 proc-supply = <&mt6397_vpca15_reg>;
69 };
70
71 &cpu1 {
72 proc-supply = <&mt6397_vpca15_reg>;
73 };
74
75 &cpu2 {
76 proc-supply = <&da9211_vcpu_reg>;
77 sram-supply = <&mt6397_vsramca7_reg>;
78 };
79
80 &cpu3 {
81 proc-supply = <&da9211_vcpu_reg>;
82 sram-supply = <&mt6397_vsramca7_reg>;
83 };
diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h
index e9e4c52f3685..64dc9f547fb6 100644
--- a/arch/powerpc/include/asm/opal-api.h
+++ b/arch/powerpc/include/asm/opal-api.h
@@ -361,6 +361,7 @@ enum opal_msg_type {
361 OPAL_MSG_HMI_EVT, 361 OPAL_MSG_HMI_EVT,
362 OPAL_MSG_DPO, 362 OPAL_MSG_DPO,
363 OPAL_MSG_PRD, 363 OPAL_MSG_PRD,
364 OPAL_MSG_OCC,
364 OPAL_MSG_TYPE_MAX, 365 OPAL_MSG_TYPE_MAX,
365}; 366};
366 367
@@ -700,6 +701,17 @@ struct opal_prd_msg_header {
700 701
701struct opal_prd_msg; 702struct opal_prd_msg;
702 703
704#define OCC_RESET 0
705#define OCC_LOAD 1
706#define OCC_THROTTLE 2
707#define OCC_MAX_THROTTLE_STATUS 5
708
709struct opal_occ_msg {
710 __be64 type;
711 __be64 chip;
712 __be64 throttle_status;
713};
714
703/* 715/*
704 * SG entries 716 * SG entries
705 * 717 *
diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c
index 53cfe8ba9799..bb01dea39fdc 100644
--- a/drivers/acpi/processor_perflib.c
+++ b/drivers/acpi/processor_perflib.c
@@ -83,7 +83,7 @@ static int acpi_processor_ppc_notifier(struct notifier_block *nb,
83 if (ignore_ppc) 83 if (ignore_ppc)
84 return 0; 84 return 0;
85 85
86 if (event != CPUFREQ_INCOMPATIBLE) 86 if (event != CPUFREQ_ADJUST)
87 return 0; 87 return 0;
88 88
89 mutex_lock(&performance_mutex); 89 mutex_lock(&performance_mutex);
@@ -780,9 +780,7 @@ acpi_processor_register_performance(struct acpi_processor_performance
780 780
781EXPORT_SYMBOL(acpi_processor_register_performance); 781EXPORT_SYMBOL(acpi_processor_register_performance);
782 782
783void 783void acpi_processor_unregister_performance(unsigned int cpu)
784acpi_processor_unregister_performance(struct acpi_processor_performance
785 *performance, unsigned int cpu)
786{ 784{
787 struct acpi_processor *pr; 785 struct acpi_processor *pr;
788 786
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index cc8a71c267b8..2bacf24a19a9 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -130,6 +130,13 @@ config ARM_KIRKWOOD_CPUFREQ
130 This adds the CPUFreq driver for Marvell Kirkwood 130 This adds the CPUFreq driver for Marvell Kirkwood
131 SoCs. 131 SoCs.
132 132
133config ARM_MT8173_CPUFREQ
134 bool "Mediatek MT8173 CPUFreq support"
135 depends on ARCH_MEDIATEK && REGULATOR
136 select PM_OPP
137 help
138 This adds the CPUFreq driver support for Mediatek MT8173 SoC.
139
133config ARM_OMAP2PLUS_CPUFREQ 140config ARM_OMAP2PLUS_CPUFREQ
134 bool "TI OMAP2+" 141 bool "TI OMAP2+"
135 depends on ARCH_OMAP2PLUS 142 depends on ARCH_OMAP2PLUS
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 2169bf792db7..9c75fafd2901 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -62,6 +62,7 @@ obj-$(CONFIG_ARM_HISI_ACPU_CPUFREQ) += hisi-acpu-cpufreq.o
62obj-$(CONFIG_ARM_IMX6Q_CPUFREQ) += imx6q-cpufreq.o 62obj-$(CONFIG_ARM_IMX6Q_CPUFREQ) += imx6q-cpufreq.o
63obj-$(CONFIG_ARM_INTEGRATOR) += integrator-cpufreq.o 63obj-$(CONFIG_ARM_INTEGRATOR) += integrator-cpufreq.o
64obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ) += kirkwood-cpufreq.o 64obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ) += kirkwood-cpufreq.o
65obj-$(CONFIG_ARM_MT8173_CPUFREQ) += mt8173-cpufreq.o
65obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ) += omap-cpufreq.o 66obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ) += omap-cpufreq.o
66obj-$(CONFIG_ARM_PXA2xx_CPUFREQ) += pxa2xx-cpufreq.o 67obj-$(CONFIG_ARM_PXA2xx_CPUFREQ) += pxa2xx-cpufreq.o
67obj-$(CONFIG_PXA3xx) += pxa3xx-cpufreq.o 68obj-$(CONFIG_PXA3xx) += pxa3xx-cpufreq.o
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
index 0136dfcdabf0..15b921a9248c 100644
--- a/drivers/cpufreq/acpi-cpufreq.c
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -65,18 +65,21 @@ enum {
65#define MSR_K7_HWCR_CPB_DIS (1ULL << 25) 65#define MSR_K7_HWCR_CPB_DIS (1ULL << 25)
66 66
67struct acpi_cpufreq_data { 67struct acpi_cpufreq_data {
68 struct acpi_processor_performance *acpi_data;
69 struct cpufreq_frequency_table *freq_table; 68 struct cpufreq_frequency_table *freq_table;
70 unsigned int resume; 69 unsigned int resume;
71 unsigned int cpu_feature; 70 unsigned int cpu_feature;
71 unsigned int acpi_perf_cpu;
72 cpumask_var_t freqdomain_cpus; 72 cpumask_var_t freqdomain_cpus;
73}; 73};
74 74
75static DEFINE_PER_CPU(struct acpi_cpufreq_data *, acfreq_data);
76
77/* acpi_perf_data is a pointer to percpu data. */ 75/* acpi_perf_data is a pointer to percpu data. */
78static struct acpi_processor_performance __percpu *acpi_perf_data; 76static struct acpi_processor_performance __percpu *acpi_perf_data;
79 77
78static inline struct acpi_processor_performance *to_perf_data(struct acpi_cpufreq_data *data)
79{
80 return per_cpu_ptr(acpi_perf_data, data->acpi_perf_cpu);
81}
82
80static struct cpufreq_driver acpi_cpufreq_driver; 83static struct cpufreq_driver acpi_cpufreq_driver;
81 84
82static unsigned int acpi_pstate_strict; 85static unsigned int acpi_pstate_strict;
@@ -144,7 +147,7 @@ static int _store_boost(int val)
144 147
145static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) 148static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)
146{ 149{
147 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); 150 struct acpi_cpufreq_data *data = policy->driver_data;
148 151
149 return cpufreq_show_cpus(data->freqdomain_cpus, buf); 152 return cpufreq_show_cpus(data->freqdomain_cpus, buf);
150} 153}
@@ -202,7 +205,7 @@ static unsigned extract_io(u32 value, struct acpi_cpufreq_data *data)
202 struct acpi_processor_performance *perf; 205 struct acpi_processor_performance *perf;
203 int i; 206 int i;
204 207
205 perf = data->acpi_data; 208 perf = to_perf_data(data);
206 209
207 for (i = 0; i < perf->state_count; i++) { 210 for (i = 0; i < perf->state_count; i++) {
208 if (value == perf->states[i].status) 211 if (value == perf->states[i].status)
@@ -221,7 +224,7 @@ static unsigned extract_msr(u32 msr, struct acpi_cpufreq_data *data)
221 else 224 else
222 msr &= INTEL_MSR_RANGE; 225 msr &= INTEL_MSR_RANGE;
223 226
224 perf = data->acpi_data; 227 perf = to_perf_data(data);
225 228
226 cpufreq_for_each_entry(pos, data->freq_table) 229 cpufreq_for_each_entry(pos, data->freq_table)
227 if (msr == perf->states[pos->driver_data].status) 230 if (msr == perf->states[pos->driver_data].status)
@@ -327,7 +330,8 @@ static void drv_write(struct drv_cmd *cmd)
327 put_cpu(); 330 put_cpu();
328} 331}
329 332
330static u32 get_cur_val(const struct cpumask *mask) 333static u32
334get_cur_val(const struct cpumask *mask, struct acpi_cpufreq_data *data)
331{ 335{
332 struct acpi_processor_performance *perf; 336 struct acpi_processor_performance *perf;
333 struct drv_cmd cmd; 337 struct drv_cmd cmd;
@@ -335,7 +339,7 @@ static u32 get_cur_val(const struct cpumask *mask)
335 if (unlikely(cpumask_empty(mask))) 339 if (unlikely(cpumask_empty(mask)))
336 return 0; 340 return 0;
337 341
338 switch (per_cpu(acfreq_data, cpumask_first(mask))->cpu_feature) { 342 switch (data->cpu_feature) {
339 case SYSTEM_INTEL_MSR_CAPABLE: 343 case SYSTEM_INTEL_MSR_CAPABLE:
340 cmd.type = SYSTEM_INTEL_MSR_CAPABLE; 344 cmd.type = SYSTEM_INTEL_MSR_CAPABLE;
341 cmd.addr.msr.reg = MSR_IA32_PERF_CTL; 345 cmd.addr.msr.reg = MSR_IA32_PERF_CTL;
@@ -346,7 +350,7 @@ static u32 get_cur_val(const struct cpumask *mask)
346 break; 350 break;
347 case SYSTEM_IO_CAPABLE: 351 case SYSTEM_IO_CAPABLE:
348 cmd.type = SYSTEM_IO_CAPABLE; 352 cmd.type = SYSTEM_IO_CAPABLE;
349 perf = per_cpu(acfreq_data, cpumask_first(mask))->acpi_data; 353 perf = to_perf_data(data);
350 cmd.addr.io.port = perf->control_register.address; 354 cmd.addr.io.port = perf->control_register.address;
351 cmd.addr.io.bit_width = perf->control_register.bit_width; 355 cmd.addr.io.bit_width = perf->control_register.bit_width;
352 break; 356 break;
@@ -364,19 +368,24 @@ static u32 get_cur_val(const struct cpumask *mask)
364 368
365static unsigned int get_cur_freq_on_cpu(unsigned int cpu) 369static unsigned int get_cur_freq_on_cpu(unsigned int cpu)
366{ 370{
367 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, cpu); 371 struct acpi_cpufreq_data *data;
372 struct cpufreq_policy *policy;
368 unsigned int freq; 373 unsigned int freq;
369 unsigned int cached_freq; 374 unsigned int cached_freq;
370 375
371 pr_debug("get_cur_freq_on_cpu (%d)\n", cpu); 376 pr_debug("get_cur_freq_on_cpu (%d)\n", cpu);
372 377
373 if (unlikely(data == NULL || 378 policy = cpufreq_cpu_get(cpu);
374 data->acpi_data == NULL || data->freq_table == NULL)) { 379 if (unlikely(!policy))
375 return 0; 380 return 0;
376 }
377 381
378 cached_freq = data->freq_table[data->acpi_data->state].frequency; 382 data = policy->driver_data;
379 freq = extract_freq(get_cur_val(cpumask_of(cpu)), data); 383 cpufreq_cpu_put(policy);
384 if (unlikely(!data || !data->freq_table))
385 return 0;
386
387 cached_freq = data->freq_table[to_perf_data(data)->state].frequency;
388 freq = extract_freq(get_cur_val(cpumask_of(cpu), data), data);
380 if (freq != cached_freq) { 389 if (freq != cached_freq) {
381 /* 390 /*
382 * The dreaded BIOS frequency change behind our back. 391 * The dreaded BIOS frequency change behind our back.
@@ -397,7 +406,7 @@ static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq,
397 unsigned int i; 406 unsigned int i;
398 407
399 for (i = 0; i < 100; i++) { 408 for (i = 0; i < 100; i++) {
400 cur_freq = extract_freq(get_cur_val(mask), data); 409 cur_freq = extract_freq(get_cur_val(mask, data), data);
401 if (cur_freq == freq) 410 if (cur_freq == freq)
402 return 1; 411 return 1;
403 udelay(10); 412 udelay(10);
@@ -408,18 +417,17 @@ static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq,
408static int acpi_cpufreq_target(struct cpufreq_policy *policy, 417static int acpi_cpufreq_target(struct cpufreq_policy *policy,
409 unsigned int index) 418 unsigned int index)
410{ 419{
411 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); 420 struct acpi_cpufreq_data *data = policy->driver_data;
412 struct acpi_processor_performance *perf; 421 struct acpi_processor_performance *perf;
413 struct drv_cmd cmd; 422 struct drv_cmd cmd;
414 unsigned int next_perf_state = 0; /* Index into perf table */ 423 unsigned int next_perf_state = 0; /* Index into perf table */
415 int result = 0; 424 int result = 0;
416 425
417 if (unlikely(data == NULL || 426 if (unlikely(data == NULL || data->freq_table == NULL)) {
418 data->acpi_data == NULL || data->freq_table == NULL)) {
419 return -ENODEV; 427 return -ENODEV;
420 } 428 }
421 429
422 perf = data->acpi_data; 430 perf = to_perf_data(data);
423 next_perf_state = data->freq_table[index].driver_data; 431 next_perf_state = data->freq_table[index].driver_data;
424 if (perf->state == next_perf_state) { 432 if (perf->state == next_perf_state) {
425 if (unlikely(data->resume)) { 433 if (unlikely(data->resume)) {
@@ -482,8 +490,9 @@ out:
482static unsigned long 490static unsigned long
483acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu) 491acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu)
484{ 492{
485 struct acpi_processor_performance *perf = data->acpi_data; 493 struct acpi_processor_performance *perf;
486 494
495 perf = to_perf_data(data);
487 if (cpu_khz) { 496 if (cpu_khz) {
488 /* search the closest match to cpu_khz */ 497 /* search the closest match to cpu_khz */
489 unsigned int i; 498 unsigned int i;
@@ -672,17 +681,17 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
672 goto err_free; 681 goto err_free;
673 } 682 }
674 683
675 data->acpi_data = per_cpu_ptr(acpi_perf_data, cpu); 684 perf = per_cpu_ptr(acpi_perf_data, cpu);
676 per_cpu(acfreq_data, cpu) = data; 685 data->acpi_perf_cpu = cpu;
686 policy->driver_data = data;
677 687
678 if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) 688 if (cpu_has(c, X86_FEATURE_CONSTANT_TSC))
679 acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS; 689 acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS;
680 690
681 result = acpi_processor_register_performance(data->acpi_data, cpu); 691 result = acpi_processor_register_performance(perf, cpu);
682 if (result) 692 if (result)
683 goto err_free_mask; 693 goto err_free_mask;
684 694
685 perf = data->acpi_data;
686 policy->shared_type = perf->shared_type; 695 policy->shared_type = perf->shared_type;
687 696
688 /* 697 /*
@@ -838,26 +847,25 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
838err_freqfree: 847err_freqfree:
839 kfree(data->freq_table); 848 kfree(data->freq_table);
840err_unreg: 849err_unreg:
841 acpi_processor_unregister_performance(perf, cpu); 850 acpi_processor_unregister_performance(cpu);
842err_free_mask: 851err_free_mask:
843 free_cpumask_var(data->freqdomain_cpus); 852 free_cpumask_var(data->freqdomain_cpus);
844err_free: 853err_free:
845 kfree(data); 854 kfree(data);
846 per_cpu(acfreq_data, cpu) = NULL; 855 policy->driver_data = NULL;
847 856
848 return result; 857 return result;
849} 858}
850 859
851static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) 860static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy)
852{ 861{
853 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); 862 struct acpi_cpufreq_data *data = policy->driver_data;
854 863
855 pr_debug("acpi_cpufreq_cpu_exit\n"); 864 pr_debug("acpi_cpufreq_cpu_exit\n");
856 865
857 if (data) { 866 if (data) {
858 per_cpu(acfreq_data, policy->cpu) = NULL; 867 policy->driver_data = NULL;
859 acpi_processor_unregister_performance(data->acpi_data, 868 acpi_processor_unregister_performance(data->acpi_perf_cpu);
860 policy->cpu);
861 free_cpumask_var(data->freqdomain_cpus); 869 free_cpumask_var(data->freqdomain_cpus);
862 kfree(data->freq_table); 870 kfree(data->freq_table);
863 kfree(data); 871 kfree(data);
@@ -868,7 +876,7 @@ static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy)
868 876
869static int acpi_cpufreq_resume(struct cpufreq_policy *policy) 877static int acpi_cpufreq_resume(struct cpufreq_policy *policy)
870{ 878{
871 struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); 879 struct acpi_cpufreq_data *data = policy->driver_data;
872 880
873 pr_debug("acpi_cpufreq_resume\n"); 881 pr_debug("acpi_cpufreq_resume\n");
874 882
@@ -880,7 +888,9 @@ static int acpi_cpufreq_resume(struct cpufreq_policy *policy)
880static struct freq_attr *acpi_cpufreq_attr[] = { 888static struct freq_attr *acpi_cpufreq_attr[] = {
881 &cpufreq_freq_attr_scaling_available_freqs, 889 &cpufreq_freq_attr_scaling_available_freqs,
882 &freqdomain_cpus, 890 &freqdomain_cpus,
883 NULL, /* this is a placeholder for cpb, do not remove */ 891#ifdef CONFIG_X86_ACPI_CPUFREQ_CPB
892 &cpb,
893#endif
884 NULL, 894 NULL,
885}; 895};
886 896
@@ -953,17 +963,16 @@ static int __init acpi_cpufreq_init(void)
953 * only if configured. This is considered legacy code, which 963 * only if configured. This is considered legacy code, which
954 * will probably be removed at some point in the future. 964 * will probably be removed at some point in the future.
955 */ 965 */
956 if (check_amd_hwpstate_cpu(0)) { 966 if (!check_amd_hwpstate_cpu(0)) {
957 struct freq_attr **iter; 967 struct freq_attr **attr;
958
959 pr_debug("adding sysfs entry for cpb\n");
960 968
961 for (iter = acpi_cpufreq_attr; *iter != NULL; iter++) 969 pr_debug("CPB unsupported, do not expose it\n");
962 ;
963 970
964 /* make sure there is a terminator behind it */ 971 for (attr = acpi_cpufreq_attr; *attr; attr++)
965 if (iter[1] == NULL) 972 if (*attr == &cpb) {
966 *iter = &cpb; 973 *attr = NULL;
974 break;
975 }
967 } 976 }
968#endif 977#endif
969 acpi_cpufreq_boost_init(); 978 acpi_cpufreq_boost_init();
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 7a3c30c4336f..a05cc75cc45d 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -112,12 +112,6 @@ static inline bool has_target(void)
112 return cpufreq_driver->target_index || cpufreq_driver->target; 112 return cpufreq_driver->target_index || cpufreq_driver->target;
113} 113}
114 114
115/*
116 * rwsem to guarantee that cpufreq driver module doesn't unload during critical
117 * sections
118 */
119static DECLARE_RWSEM(cpufreq_rwsem);
120
121/* internal prototypes */ 115/* internal prototypes */
122static int __cpufreq_governor(struct cpufreq_policy *policy, 116static int __cpufreq_governor(struct cpufreq_policy *policy,
123 unsigned int event); 117 unsigned int event);
@@ -277,10 +271,6 @@ EXPORT_SYMBOL_GPL(cpufreq_generic_get);
277 * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be 271 * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
278 * freed as that depends on the kobj count. 272 * freed as that depends on the kobj count.
279 * 273 *
280 * It also takes a read-lock of 'cpufreq_rwsem' and doesn't put it back if a
281 * valid policy is found. This is done to make sure the driver doesn't get
282 * unregistered while the policy is being used.
283 *
284 * Return: A valid policy on success, otherwise NULL on failure. 274 * Return: A valid policy on success, otherwise NULL on failure.
285 */ 275 */
286struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) 276struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
@@ -291,9 +281,6 @@ struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
291 if (WARN_ON(cpu >= nr_cpu_ids)) 281 if (WARN_ON(cpu >= nr_cpu_ids))
292 return NULL; 282 return NULL;
293 283
294 if (!down_read_trylock(&cpufreq_rwsem))
295 return NULL;
296
297 /* get the cpufreq driver */ 284 /* get the cpufreq driver */
298 read_lock_irqsave(&cpufreq_driver_lock, flags); 285 read_lock_irqsave(&cpufreq_driver_lock, flags);
299 286
@@ -306,9 +293,6 @@ struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
306 293
307 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 294 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
308 295
309 if (!policy)
310 up_read(&cpufreq_rwsem);
311
312 return policy; 296 return policy;
313} 297}
314EXPORT_SYMBOL_GPL(cpufreq_cpu_get); 298EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
@@ -320,13 +304,10 @@ EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
320 * 304 *
321 * This decrements the kobject reference count incremented earlier by calling 305 * This decrements the kobject reference count incremented earlier by calling
322 * cpufreq_cpu_get(). 306 * cpufreq_cpu_get().
323 *
324 * It also drops the read-lock of 'cpufreq_rwsem' taken at cpufreq_cpu_get().
325 */ 307 */
326void cpufreq_cpu_put(struct cpufreq_policy *policy) 308void cpufreq_cpu_put(struct cpufreq_policy *policy)
327{ 309{
328 kobject_put(&policy->kobj); 310 kobject_put(&policy->kobj);
329 up_read(&cpufreq_rwsem);
330} 311}
331EXPORT_SYMBOL_GPL(cpufreq_cpu_put); 312EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
332 313
@@ -539,9 +520,6 @@ static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
539{ 520{
540 int err = -EINVAL; 521 int err = -EINVAL;
541 522
542 if (!cpufreq_driver)
543 goto out;
544
545 if (cpufreq_driver->setpolicy) { 523 if (cpufreq_driver->setpolicy) {
546 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) { 524 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
547 *policy = CPUFREQ_POLICY_PERFORMANCE; 525 *policy = CPUFREQ_POLICY_PERFORMANCE;
@@ -576,7 +554,6 @@ static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
576 554
577 mutex_unlock(&cpufreq_governor_mutex); 555 mutex_unlock(&cpufreq_governor_mutex);
578 } 556 }
579out:
580 return err; 557 return err;
581} 558}
582 559
@@ -625,9 +602,7 @@ static ssize_t store_##file_name \
625 int ret, temp; \ 602 int ret, temp; \
626 struct cpufreq_policy new_policy; \ 603 struct cpufreq_policy new_policy; \
627 \ 604 \
628 ret = cpufreq_get_policy(&new_policy, policy->cpu); \ 605 memcpy(&new_policy, policy, sizeof(*policy)); \
629 if (ret) \
630 return -EINVAL; \
631 \ 606 \
632 ret = sscanf(buf, "%u", &new_policy.object); \ 607 ret = sscanf(buf, "%u", &new_policy.object); \
633 if (ret != 1) \ 608 if (ret != 1) \
@@ -681,9 +656,7 @@ static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
681 char str_governor[16]; 656 char str_governor[16];
682 struct cpufreq_policy new_policy; 657 struct cpufreq_policy new_policy;
683 658
684 ret = cpufreq_get_policy(&new_policy, policy->cpu); 659 memcpy(&new_policy, policy, sizeof(*policy));
685 if (ret)
686 return ret;
687 660
688 ret = sscanf(buf, "%15s", str_governor); 661 ret = sscanf(buf, "%15s", str_governor);
689 if (ret != 1) 662 if (ret != 1)
@@ -694,14 +667,7 @@ static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
694 return -EINVAL; 667 return -EINVAL;
695 668
696 ret = cpufreq_set_policy(policy, &new_policy); 669 ret = cpufreq_set_policy(policy, &new_policy);
697 670 return ret ? ret : count;
698 policy->user_policy.policy = policy->policy;
699 policy->user_policy.governor = policy->governor;
700
701 if (ret)
702 return ret;
703 else
704 return count;
705} 671}
706 672
707/** 673/**
@@ -851,9 +817,6 @@ static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
851 struct freq_attr *fattr = to_attr(attr); 817 struct freq_attr *fattr = to_attr(attr);
852 ssize_t ret; 818 ssize_t ret;
853 819
854 if (!down_read_trylock(&cpufreq_rwsem))
855 return -EINVAL;
856
857 down_read(&policy->rwsem); 820 down_read(&policy->rwsem);
858 821
859 if (fattr->show) 822 if (fattr->show)
@@ -862,7 +825,6 @@ static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
862 ret = -EIO; 825 ret = -EIO;
863 826
864 up_read(&policy->rwsem); 827 up_read(&policy->rwsem);
865 up_read(&cpufreq_rwsem);
866 828
867 return ret; 829 return ret;
868} 830}
@@ -879,9 +841,6 @@ static ssize_t store(struct kobject *kobj, struct attribute *attr,
879 if (!cpu_online(policy->cpu)) 841 if (!cpu_online(policy->cpu))
880 goto unlock; 842 goto unlock;
881 843
882 if (!down_read_trylock(&cpufreq_rwsem))
883 goto unlock;
884
885 down_write(&policy->rwsem); 844 down_write(&policy->rwsem);
886 845
887 /* Updating inactive policies is invalid, so avoid doing that. */ 846 /* Updating inactive policies is invalid, so avoid doing that. */
@@ -897,8 +856,6 @@ static ssize_t store(struct kobject *kobj, struct attribute *attr,
897 856
898unlock_policy_rwsem: 857unlock_policy_rwsem:
899 up_write(&policy->rwsem); 858 up_write(&policy->rwsem);
900
901 up_read(&cpufreq_rwsem);
902unlock: 859unlock:
903 put_online_cpus(); 860 put_online_cpus();
904 861
@@ -1027,8 +984,7 @@ static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
1027 } 984 }
1028} 985}
1029 986
1030static int cpufreq_add_dev_interface(struct cpufreq_policy *policy, 987static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
1031 struct device *dev)
1032{ 988{
1033 struct freq_attr **drv_attr; 989 struct freq_attr **drv_attr;
1034 int ret = 0; 990 int ret = 0;
@@ -1060,11 +1016,10 @@ static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
1060 return cpufreq_add_dev_symlink(policy); 1016 return cpufreq_add_dev_symlink(policy);
1061} 1017}
1062 1018
1063static void cpufreq_init_policy(struct cpufreq_policy *policy) 1019static int cpufreq_init_policy(struct cpufreq_policy *policy)
1064{ 1020{
1065 struct cpufreq_governor *gov = NULL; 1021 struct cpufreq_governor *gov = NULL;
1066 struct cpufreq_policy new_policy; 1022 struct cpufreq_policy new_policy;
1067 int ret = 0;
1068 1023
1069 memcpy(&new_policy, policy, sizeof(*policy)); 1024 memcpy(&new_policy, policy, sizeof(*policy));
1070 1025
@@ -1083,16 +1038,10 @@ static void cpufreq_init_policy(struct cpufreq_policy *policy)
1083 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL); 1038 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
1084 1039
1085 /* set default policy */ 1040 /* set default policy */
1086 ret = cpufreq_set_policy(policy, &new_policy); 1041 return cpufreq_set_policy(policy, &new_policy);
1087 if (ret) {
1088 pr_debug("setting policy failed\n");
1089 if (cpufreq_driver->exit)
1090 cpufreq_driver->exit(policy);
1091 }
1092} 1042}
1093 1043
1094static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, 1044static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1095 unsigned int cpu, struct device *dev)
1096{ 1045{
1097 int ret = 0; 1046 int ret = 0;
1098 1047
@@ -1126,33 +1075,15 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
1126 return 0; 1075 return 0;
1127} 1076}
1128 1077
1129static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu) 1078static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1130{
1131 struct cpufreq_policy *policy;
1132 unsigned long flags;
1133
1134 read_lock_irqsave(&cpufreq_driver_lock, flags);
1135 policy = per_cpu(cpufreq_cpu_data, cpu);
1136 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1137
1138 if (likely(policy)) {
1139 /* Policy should be inactive here */
1140 WARN_ON(!policy_is_inactive(policy));
1141
1142 down_write(&policy->rwsem);
1143 policy->cpu = cpu;
1144 policy->governor = NULL;
1145 up_write(&policy->rwsem);
1146 }
1147
1148 return policy;
1149}
1150
1151static struct cpufreq_policy *cpufreq_policy_alloc(struct device *dev)
1152{ 1079{
1080 struct device *dev = get_cpu_device(cpu);
1153 struct cpufreq_policy *policy; 1081 struct cpufreq_policy *policy;
1154 int ret; 1082 int ret;
1155 1083
1084 if (WARN_ON(!dev))
1085 return NULL;
1086
1156 policy = kzalloc(sizeof(*policy), GFP_KERNEL); 1087 policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1157 if (!policy) 1088 if (!policy)
1158 return NULL; 1089 return NULL;
@@ -1180,10 +1111,10 @@ static struct cpufreq_policy *cpufreq_policy_alloc(struct device *dev)
1180 init_completion(&policy->kobj_unregister); 1111 init_completion(&policy->kobj_unregister);
1181 INIT_WORK(&policy->update, handle_update); 1112 INIT_WORK(&policy->update, handle_update);
1182 1113
1183 policy->cpu = dev->id; 1114 policy->cpu = cpu;
1184 1115
1185 /* Set this once on allocation */ 1116 /* Set this once on allocation */
1186 policy->kobj_cpu = dev->id; 1117 policy->kobj_cpu = cpu;
1187 1118
1188 return policy; 1119 return policy;
1189 1120
@@ -1245,59 +1176,34 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify)
1245 kfree(policy); 1176 kfree(policy);
1246} 1177}
1247 1178
1248/** 1179static int cpufreq_online(unsigned int cpu)
1249 * cpufreq_add_dev - add a CPU device
1250 *
1251 * Adds the cpufreq interface for a CPU device.
1252 *
1253 * The Oracle says: try running cpufreq registration/unregistration concurrently
1254 * with with cpu hotplugging and all hell will break loose. Tried to clean this
1255 * mess up, but more thorough testing is needed. - Mathieu
1256 */
1257static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1258{ 1180{
1259 unsigned int j, cpu = dev->id;
1260 int ret = -ENOMEM;
1261 struct cpufreq_policy *policy; 1181 struct cpufreq_policy *policy;
1182 bool new_policy;
1262 unsigned long flags; 1183 unsigned long flags;
1263 bool recover_policy = !sif; 1184 unsigned int j;
1264 1185 int ret;
1265 pr_debug("adding CPU %u\n", cpu);
1266
1267 if (cpu_is_offline(cpu)) {
1268 /*
1269 * Only possible if we are here from the subsys_interface add
1270 * callback. A hotplug notifier will follow and we will handle
1271 * it as CPU online then. For now, just create the sysfs link,
1272 * unless there is no policy or the link is already present.
1273 */
1274 policy = per_cpu(cpufreq_cpu_data, cpu);
1275 return policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus)
1276 ? add_cpu_dev_symlink(policy, cpu) : 0;
1277 }
1278 1186
1279 if (!down_read_trylock(&cpufreq_rwsem)) 1187 pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1280 return 0;
1281 1188
1282 /* Check if this CPU already has a policy to manage it */ 1189 /* Check if this CPU already has a policy to manage it */
1283 policy = per_cpu(cpufreq_cpu_data, cpu); 1190 policy = per_cpu(cpufreq_cpu_data, cpu);
1284 if (policy && !policy_is_inactive(policy)) { 1191 if (policy) {
1285 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); 1192 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1286 ret = cpufreq_add_policy_cpu(policy, cpu, dev); 1193 if (!policy_is_inactive(policy))
1287 up_read(&cpufreq_rwsem); 1194 return cpufreq_add_policy_cpu(policy, cpu);
1288 return ret;
1289 }
1290 1195
1291 /* 1196 /* This is the only online CPU for the policy. Start over. */
1292 * Restore the saved policy when doing light-weight init and fall back 1197 new_policy = false;
1293 * to the full init if that fails. 1198 down_write(&policy->rwsem);
1294 */ 1199 policy->cpu = cpu;
1295 policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL; 1200 policy->governor = NULL;
1296 if (!policy) { 1201 up_write(&policy->rwsem);
1297 recover_policy = false; 1202 } else {
1298 policy = cpufreq_policy_alloc(dev); 1203 new_policy = true;
1204 policy = cpufreq_policy_alloc(cpu);
1299 if (!policy) 1205 if (!policy)
1300 goto nomem_out; 1206 return -ENOMEM;
1301 } 1207 }
1302 1208
1303 cpumask_copy(policy->cpus, cpumask_of(cpu)); 1209 cpumask_copy(policy->cpus, cpumask_of(cpu));
@@ -1308,17 +1214,17 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1308 ret = cpufreq_driver->init(policy); 1214 ret = cpufreq_driver->init(policy);
1309 if (ret) { 1215 if (ret) {
1310 pr_debug("initialization failed\n"); 1216 pr_debug("initialization failed\n");
1311 goto err_set_policy_cpu; 1217 goto out_free_policy;
1312 } 1218 }
1313 1219
1314 down_write(&policy->rwsem); 1220 down_write(&policy->rwsem);
1315 1221
1316 /* related cpus should atleast have policy->cpus */ 1222 if (new_policy) {
1317 cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus); 1223 /* related_cpus should at least include policy->cpus. */
1318 1224 cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1319 /* Remember which CPUs have been present at the policy creation time. */ 1225 /* Remember CPUs present at the policy creation time. */
1320 if (!recover_policy)
1321 cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask); 1226 cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask);
1227 }
1322 1228
1323 /* 1229 /*
1324 * affected cpus must always be the one, which are online. We aren't 1230 * affected cpus must always be the one, which are online. We aren't
@@ -1326,7 +1232,7 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1326 */ 1232 */
1327 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); 1233 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1328 1234
1329 if (!recover_policy) { 1235 if (new_policy) {
1330 policy->user_policy.min = policy->min; 1236 policy->user_policy.min = policy->min;
1331 policy->user_policy.max = policy->max; 1237 policy->user_policy.max = policy->max;
1332 1238
@@ -1340,7 +1246,7 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1340 policy->cur = cpufreq_driver->get(policy->cpu); 1246 policy->cur = cpufreq_driver->get(policy->cpu);
1341 if (!policy->cur) { 1247 if (!policy->cur) {
1342 pr_err("%s: ->get() failed\n", __func__); 1248 pr_err("%s: ->get() failed\n", __func__);
1343 goto err_get_freq; 1249 goto out_exit_policy;
1344 } 1250 }
1345 } 1251 }
1346 1252
@@ -1387,10 +1293,10 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1387 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1293 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1388 CPUFREQ_START, policy); 1294 CPUFREQ_START, policy);
1389 1295
1390 if (!recover_policy) { 1296 if (new_policy) {
1391 ret = cpufreq_add_dev_interface(policy, dev); 1297 ret = cpufreq_add_dev_interface(policy);
1392 if (ret) 1298 if (ret)
1393 goto err_out_unregister; 1299 goto out_exit_policy;
1394 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1300 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1395 CPUFREQ_CREATE_POLICY, policy); 1301 CPUFREQ_CREATE_POLICY, policy);
1396 1302
@@ -1399,18 +1305,19 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1399 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1305 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1400 } 1306 }
1401 1307
1402 cpufreq_init_policy(policy); 1308 ret = cpufreq_init_policy(policy);
1403 1309 if (ret) {
1404 if (!recover_policy) { 1310 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1405 policy->user_policy.policy = policy->policy; 1311 __func__, cpu, ret);
1406 policy->user_policy.governor = policy->governor; 1312 /* cpufreq_policy_free() will notify based on this */
1313 new_policy = false;
1314 goto out_exit_policy;
1407 } 1315 }
1316
1408 up_write(&policy->rwsem); 1317 up_write(&policy->rwsem);
1409 1318
1410 kobject_uevent(&policy->kobj, KOBJ_ADD); 1319 kobject_uevent(&policy->kobj, KOBJ_ADD);
1411 1320
1412 up_read(&cpufreq_rwsem);
1413
1414 /* Callback for handling stuff after policy is ready */ 1321 /* Callback for handling stuff after policy is ready */
1415 if (cpufreq_driver->ready) 1322 if (cpufreq_driver->ready)
1416 cpufreq_driver->ready(policy); 1323 cpufreq_driver->ready(policy);
@@ -1419,24 +1326,47 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1419 1326
1420 return 0; 1327 return 0;
1421 1328
1422err_out_unregister: 1329out_exit_policy:
1423err_get_freq:
1424 up_write(&policy->rwsem); 1330 up_write(&policy->rwsem);
1425 1331
1426 if (cpufreq_driver->exit) 1332 if (cpufreq_driver->exit)
1427 cpufreq_driver->exit(policy); 1333 cpufreq_driver->exit(policy);
1428err_set_policy_cpu: 1334out_free_policy:
1429 cpufreq_policy_free(policy, recover_policy); 1335 cpufreq_policy_free(policy, !new_policy);
1430nomem_out: 1336 return ret;
1431 up_read(&cpufreq_rwsem); 1337}
1338
1339/**
1340 * cpufreq_add_dev - the cpufreq interface for a CPU device.
1341 * @dev: CPU device.
1342 * @sif: Subsystem interface structure pointer (not used)
1343 */
1344static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1345{
1346 unsigned cpu = dev->id;
1347 int ret;
1348
1349 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1350
1351 if (cpu_online(cpu)) {
1352 ret = cpufreq_online(cpu);
1353 } else {
1354 /*
1355 * A hotplug notifier will follow and we will handle it as CPU
1356 * online then. For now, just create the sysfs link, unless
1357 * there is no policy or the link is already present.
1358 */
1359 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1360
1361 ret = policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus)
1362 ? add_cpu_dev_symlink(policy, cpu) : 0;
1363 }
1432 1364
1433 return ret; 1365 return ret;
1434} 1366}
1435 1367
1436static int __cpufreq_remove_dev_prepare(struct device *dev) 1368static void cpufreq_offline_prepare(unsigned int cpu)
1437{ 1369{
1438 unsigned int cpu = dev->id;
1439 int ret = 0;
1440 struct cpufreq_policy *policy; 1370 struct cpufreq_policy *policy;
1441 1371
1442 pr_debug("%s: unregistering CPU %u\n", __func__, cpu); 1372 pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
@@ -1444,11 +1374,11 @@ static int __cpufreq_remove_dev_prepare(struct device *dev)
1444 policy = cpufreq_cpu_get_raw(cpu); 1374 policy = cpufreq_cpu_get_raw(cpu);
1445 if (!policy) { 1375 if (!policy) {
1446 pr_debug("%s: No cpu_data found\n", __func__); 1376 pr_debug("%s: No cpu_data found\n", __func__);
1447 return -EINVAL; 1377 return;
1448 } 1378 }
1449 1379
1450 if (has_target()) { 1380 if (has_target()) {
1451 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); 1381 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1452 if (ret) 1382 if (ret)
1453 pr_err("%s: Failed to stop governor\n", __func__); 1383 pr_err("%s: Failed to stop governor\n", __func__);
1454 } 1384 }
@@ -1469,7 +1399,7 @@ static int __cpufreq_remove_dev_prepare(struct device *dev)
1469 /* Start governor again for active policy */ 1399 /* Start governor again for active policy */
1470 if (!policy_is_inactive(policy)) { 1400 if (!policy_is_inactive(policy)) {
1471 if (has_target()) { 1401 if (has_target()) {
1472 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START); 1402 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1473 if (!ret) 1403 if (!ret)
1474 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 1404 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1475 1405
@@ -1479,28 +1409,24 @@ static int __cpufreq_remove_dev_prepare(struct device *dev)
1479 } else if (cpufreq_driver->stop_cpu) { 1409 } else if (cpufreq_driver->stop_cpu) {
1480 cpufreq_driver->stop_cpu(policy); 1410 cpufreq_driver->stop_cpu(policy);
1481 } 1411 }
1482
1483 return ret;
1484} 1412}
1485 1413
1486static int __cpufreq_remove_dev_finish(struct device *dev) 1414static void cpufreq_offline_finish(unsigned int cpu)
1487{ 1415{
1488 unsigned int cpu = dev->id;
1489 int ret;
1490 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 1416 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1491 1417
1492 if (!policy) { 1418 if (!policy) {
1493 pr_debug("%s: No cpu_data found\n", __func__); 1419 pr_debug("%s: No cpu_data found\n", __func__);
1494 return -EINVAL; 1420 return;
1495 } 1421 }
1496 1422
1497 /* Only proceed for inactive policies */ 1423 /* Only proceed for inactive policies */
1498 if (!policy_is_inactive(policy)) 1424 if (!policy_is_inactive(policy))
1499 return 0; 1425 return;
1500 1426
1501 /* If cpu is last user of policy, free policy */ 1427 /* If cpu is last user of policy, free policy */
1502 if (has_target()) { 1428 if (has_target()) {
1503 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); 1429 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1504 if (ret) 1430 if (ret)
1505 pr_err("%s: Failed to exit governor\n", __func__); 1431 pr_err("%s: Failed to exit governor\n", __func__);
1506 } 1432 }
@@ -1512,8 +1438,6 @@ static int __cpufreq_remove_dev_finish(struct device *dev)
1512 */ 1438 */
1513 if (cpufreq_driver->exit) 1439 if (cpufreq_driver->exit)
1514 cpufreq_driver->exit(policy); 1440 cpufreq_driver->exit(policy);
1515
1516 return 0;
1517} 1441}
1518 1442
1519/** 1443/**
@@ -1530,8 +1454,8 @@ static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1530 return 0; 1454 return 0;
1531 1455
1532 if (cpu_online(cpu)) { 1456 if (cpu_online(cpu)) {
1533 __cpufreq_remove_dev_prepare(dev); 1457 cpufreq_offline_prepare(cpu);
1534 __cpufreq_remove_dev_finish(dev); 1458 cpufreq_offline_finish(cpu);
1535 } 1459 }
1536 1460
1537 cpumask_clear_cpu(cpu, policy->real_cpus); 1461 cpumask_clear_cpu(cpu, policy->real_cpus);
@@ -2247,7 +2171,11 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2247 2171
2248 memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); 2172 memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2249 2173
2250 if (new_policy->min > policy->max || new_policy->max < policy->min) 2174 /*
2175 * This check works well when we store new min/max freq attributes,
2176 * because new_policy is a copy of policy with one field updated.
2177 */
2178 if (new_policy->min > new_policy->max)
2251 return -EINVAL; 2179 return -EINVAL;
2252 2180
2253 /* verify the cpu speed can be set within this limit */ 2181 /* verify the cpu speed can be set within this limit */
@@ -2259,10 +2187,6 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2259 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 2187 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2260 CPUFREQ_ADJUST, new_policy); 2188 CPUFREQ_ADJUST, new_policy);
2261 2189
2262 /* adjust if necessary - hardware incompatibility*/
2263 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2264 CPUFREQ_INCOMPATIBLE, new_policy);
2265
2266 /* 2190 /*
2267 * verify the cpu speed can be set within this limit, which might be 2191 * verify the cpu speed can be set within this limit, which might be
2268 * different to the first one 2192 * different to the first one
@@ -2296,16 +2220,31 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2296 old_gov = policy->governor; 2220 old_gov = policy->governor;
2297 /* end old governor */ 2221 /* end old governor */
2298 if (old_gov) { 2222 if (old_gov) {
2299 __cpufreq_governor(policy, CPUFREQ_GOV_STOP); 2223 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2224 if (ret) {
2225 /* This can happen due to race with other operations */
2226 pr_debug("%s: Failed to Stop Governor: %s (%d)\n",
2227 __func__, old_gov->name, ret);
2228 return ret;
2229 }
2230
2300 up_write(&policy->rwsem); 2231 up_write(&policy->rwsem);
2301 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); 2232 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2302 down_write(&policy->rwsem); 2233 down_write(&policy->rwsem);
2234
2235 if (ret) {
2236 pr_err("%s: Failed to Exit Governor: %s (%d)\n",
2237 __func__, old_gov->name, ret);
2238 return ret;
2239 }
2303 } 2240 }
2304 2241
2305 /* start new governor */ 2242 /* start new governor */
2306 policy->governor = new_policy->governor; 2243 policy->governor = new_policy->governor;
2307 if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) { 2244 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2308 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) 2245 if (!ret) {
2246 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
2247 if (!ret)
2309 goto out; 2248 goto out;
2310 2249
2311 up_write(&policy->rwsem); 2250 up_write(&policy->rwsem);
@@ -2317,11 +2256,13 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
2317 pr_debug("starting governor %s failed\n", policy->governor->name); 2256 pr_debug("starting governor %s failed\n", policy->governor->name);
2318 if (old_gov) { 2257 if (old_gov) {
2319 policy->governor = old_gov; 2258 policy->governor = old_gov;
2320 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT); 2259 if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT))
2321 __cpufreq_governor(policy, CPUFREQ_GOV_START); 2260 policy->governor = NULL;
2261 else
2262 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2322 } 2263 }
2323 2264
2324 return -EINVAL; 2265 return ret;
2325 2266
2326 out: 2267 out:
2327 pr_debug("governor: change or update limits\n"); 2268 pr_debug("governor: change or update limits\n");
@@ -2350,8 +2291,6 @@ int cpufreq_update_policy(unsigned int cpu)
2350 memcpy(&new_policy, policy, sizeof(*policy)); 2291 memcpy(&new_policy, policy, sizeof(*policy));
2351 new_policy.min = policy->user_policy.min; 2292 new_policy.min = policy->user_policy.min;
2352 new_policy.max = policy->user_policy.max; 2293 new_policy.max = policy->user_policy.max;
2353 new_policy.policy = policy->user_policy.policy;
2354 new_policy.governor = policy->user_policy.governor;
2355 2294
2356 /* 2295 /*
2357 * BIOS might change freq behind our back 2296 * BIOS might change freq behind our back
@@ -2387,27 +2326,23 @@ static int cpufreq_cpu_callback(struct notifier_block *nfb,
2387 unsigned long action, void *hcpu) 2326 unsigned long action, void *hcpu)
2388{ 2327{
2389 unsigned int cpu = (unsigned long)hcpu; 2328 unsigned int cpu = (unsigned long)hcpu;
2390 struct device *dev;
2391 2329
2392 dev = get_cpu_device(cpu); 2330 switch (action & ~CPU_TASKS_FROZEN) {
2393 if (dev) { 2331 case CPU_ONLINE:
2394 switch (action & ~CPU_TASKS_FROZEN) { 2332 cpufreq_online(cpu);
2395 case CPU_ONLINE: 2333 break;
2396 cpufreq_add_dev(dev, NULL);
2397 break;
2398 2334
2399 case CPU_DOWN_PREPARE: 2335 case CPU_DOWN_PREPARE:
2400 __cpufreq_remove_dev_prepare(dev); 2336 cpufreq_offline_prepare(cpu);
2401 break; 2337 break;
2402 2338
2403 case CPU_POST_DEAD: 2339 case CPU_POST_DEAD:
2404 __cpufreq_remove_dev_finish(dev); 2340 cpufreq_offline_finish(cpu);
2405 break; 2341 break;
2406 2342
2407 case CPU_DOWN_FAILED: 2343 case CPU_DOWN_FAILED:
2408 cpufreq_add_dev(dev, NULL); 2344 cpufreq_online(cpu);
2409 break; 2345 break;
2410 }
2411 } 2346 }
2412 return NOTIFY_OK; 2347 return NOTIFY_OK;
2413} 2348}
@@ -2515,10 +2450,14 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2515 2450
2516 pr_debug("trying to register driver %s\n", driver_data->name); 2451 pr_debug("trying to register driver %s\n", driver_data->name);
2517 2452
2453 /* Protect against concurrent CPU online/offline. */
2454 get_online_cpus();
2455
2518 write_lock_irqsave(&cpufreq_driver_lock, flags); 2456 write_lock_irqsave(&cpufreq_driver_lock, flags);
2519 if (cpufreq_driver) { 2457 if (cpufreq_driver) {
2520 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2458 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2521 return -EEXIST; 2459 ret = -EEXIST;
2460 goto out;
2522 } 2461 }
2523 cpufreq_driver = driver_data; 2462 cpufreq_driver = driver_data;
2524 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2463 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
@@ -2557,7 +2496,10 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2557 register_hotcpu_notifier(&cpufreq_cpu_notifier); 2496 register_hotcpu_notifier(&cpufreq_cpu_notifier);
2558 pr_debug("driver %s up and running\n", driver_data->name); 2497 pr_debug("driver %s up and running\n", driver_data->name);
2559 2498
2560 return 0; 2499out:
2500 put_online_cpus();
2501 return ret;
2502
2561err_if_unreg: 2503err_if_unreg:
2562 subsys_interface_unregister(&cpufreq_interface); 2504 subsys_interface_unregister(&cpufreq_interface);
2563err_boost_unreg: 2505err_boost_unreg:
@@ -2567,7 +2509,7 @@ err_null_driver:
2567 write_lock_irqsave(&cpufreq_driver_lock, flags); 2509 write_lock_irqsave(&cpufreq_driver_lock, flags);
2568 cpufreq_driver = NULL; 2510 cpufreq_driver = NULL;
2569 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2511 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2570 return ret; 2512 goto out;
2571} 2513}
2572EXPORT_SYMBOL_GPL(cpufreq_register_driver); 2514EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2573 2515
@@ -2588,19 +2530,20 @@ int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2588 2530
2589 pr_debug("unregistering driver %s\n", driver->name); 2531 pr_debug("unregistering driver %s\n", driver->name);
2590 2532
2533 /* Protect against concurrent cpu hotplug */
2534 get_online_cpus();
2591 subsys_interface_unregister(&cpufreq_interface); 2535 subsys_interface_unregister(&cpufreq_interface);
2592 if (cpufreq_boost_supported()) 2536 if (cpufreq_boost_supported())
2593 cpufreq_sysfs_remove_file(&boost.attr); 2537 cpufreq_sysfs_remove_file(&boost.attr);
2594 2538
2595 unregister_hotcpu_notifier(&cpufreq_cpu_notifier); 2539 unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2596 2540
2597 down_write(&cpufreq_rwsem);
2598 write_lock_irqsave(&cpufreq_driver_lock, flags); 2541 write_lock_irqsave(&cpufreq_driver_lock, flags);
2599 2542
2600 cpufreq_driver = NULL; 2543 cpufreq_driver = NULL;
2601 2544
2602 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2545 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2603 up_write(&cpufreq_rwsem); 2546 put_online_cpus();
2604 2547
2605 return 0; 2548 return 0;
2606} 2549}
diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c
index c86a10c30912..84a1506950a7 100644
--- a/drivers/cpufreq/cpufreq_conservative.c
+++ b/drivers/cpufreq/cpufreq_conservative.c
@@ -47,7 +47,7 @@ static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners,
47static void cs_check_cpu(int cpu, unsigned int load) 47static void cs_check_cpu(int cpu, unsigned int load)
48{ 48{
49 struct cs_cpu_dbs_info_s *dbs_info = &per_cpu(cs_cpu_dbs_info, cpu); 49 struct cs_cpu_dbs_info_s *dbs_info = &per_cpu(cs_cpu_dbs_info, cpu);
50 struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; 50 struct cpufreq_policy *policy = dbs_info->cdbs.shared->policy;
51 struct dbs_data *dbs_data = policy->governor_data; 51 struct dbs_data *dbs_data = policy->governor_data;
52 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 52 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
53 53
@@ -102,26 +102,15 @@ static void cs_check_cpu(int cpu, unsigned int load)
102 } 102 }
103} 103}
104 104
105static void cs_dbs_timer(struct work_struct *work) 105static unsigned int cs_dbs_timer(struct cpu_dbs_info *cdbs,
106 struct dbs_data *dbs_data, bool modify_all)
106{ 107{
107 struct cs_cpu_dbs_info_s *dbs_info = container_of(work,
108 struct cs_cpu_dbs_info_s, cdbs.work.work);
109 unsigned int cpu = dbs_info->cdbs.cur_policy->cpu;
110 struct cs_cpu_dbs_info_s *core_dbs_info = &per_cpu(cs_cpu_dbs_info,
111 cpu);
112 struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data;
113 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 108 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
114 int delay = delay_for_sampling_rate(cs_tuners->sampling_rate);
115 bool modify_all = true;
116 109
117 mutex_lock(&core_dbs_info->cdbs.timer_mutex); 110 if (modify_all)
118 if (!need_load_eval(&core_dbs_info->cdbs, cs_tuners->sampling_rate)) 111 dbs_check_cpu(dbs_data, cdbs->shared->policy->cpu);
119 modify_all = false;
120 else
121 dbs_check_cpu(dbs_data, cpu);
122 112
123 gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, delay, modify_all); 113 return delay_for_sampling_rate(cs_tuners->sampling_rate);
124 mutex_unlock(&core_dbs_info->cdbs.timer_mutex);
125} 114}
126 115
127static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 116static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
@@ -135,7 +124,7 @@ static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
135 if (!dbs_info->enable) 124 if (!dbs_info->enable)
136 return 0; 125 return 0;
137 126
138 policy = dbs_info->cdbs.cur_policy; 127 policy = dbs_info->cdbs.shared->policy;
139 128
140 /* 129 /*
141 * we only care if our internally tracked freq moves outside the 'valid' 130 * we only care if our internally tracked freq moves outside the 'valid'
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index 57a39f8a92b7..939197ffa4ac 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -32,10 +32,10 @@ static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data)
32 32
33void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) 33void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
34{ 34{
35 struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); 35 struct cpu_dbs_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
36 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 36 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
37 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 37 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
38 struct cpufreq_policy *policy; 38 struct cpufreq_policy *policy = cdbs->shared->policy;
39 unsigned int sampling_rate; 39 unsigned int sampling_rate;
40 unsigned int max_load = 0; 40 unsigned int max_load = 0;
41 unsigned int ignore_nice; 41 unsigned int ignore_nice;
@@ -60,11 +60,9 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
60 ignore_nice = cs_tuners->ignore_nice_load; 60 ignore_nice = cs_tuners->ignore_nice_load;
61 } 61 }
62 62
63 policy = cdbs->cur_policy;
64
65 /* Get Absolute Load */ 63 /* Get Absolute Load */
66 for_each_cpu(j, policy->cpus) { 64 for_each_cpu(j, policy->cpus) {
67 struct cpu_dbs_common_info *j_cdbs; 65 struct cpu_dbs_info *j_cdbs;
68 u64 cur_wall_time, cur_idle_time; 66 u64 cur_wall_time, cur_idle_time;
69 unsigned int idle_time, wall_time; 67 unsigned int idle_time, wall_time;
70 unsigned int load; 68 unsigned int load;
@@ -163,9 +161,9 @@ EXPORT_SYMBOL_GPL(dbs_check_cpu);
163static inline void __gov_queue_work(int cpu, struct dbs_data *dbs_data, 161static inline void __gov_queue_work(int cpu, struct dbs_data *dbs_data,
164 unsigned int delay) 162 unsigned int delay)
165{ 163{
166 struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); 164 struct cpu_dbs_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
167 165
168 mod_delayed_work_on(cpu, system_wq, &cdbs->work, delay); 166 mod_delayed_work_on(cpu, system_wq, &cdbs->dwork, delay);
169} 167}
170 168
171void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy, 169void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy,
@@ -199,33 +197,63 @@ EXPORT_SYMBOL_GPL(gov_queue_work);
199static inline void gov_cancel_work(struct dbs_data *dbs_data, 197static inline void gov_cancel_work(struct dbs_data *dbs_data,
200 struct cpufreq_policy *policy) 198 struct cpufreq_policy *policy)
201{ 199{
202 struct cpu_dbs_common_info *cdbs; 200 struct cpu_dbs_info *cdbs;
203 int i; 201 int i;
204 202
205 for_each_cpu(i, policy->cpus) { 203 for_each_cpu(i, policy->cpus) {
206 cdbs = dbs_data->cdata->get_cpu_cdbs(i); 204 cdbs = dbs_data->cdata->get_cpu_cdbs(i);
207 cancel_delayed_work_sync(&cdbs->work); 205 cancel_delayed_work_sync(&cdbs->dwork);
208 } 206 }
209} 207}
210 208
211/* Will return if we need to evaluate cpu load again or not */ 209/* Will return if we need to evaluate cpu load again or not */
212bool need_load_eval(struct cpu_dbs_common_info *cdbs, 210static bool need_load_eval(struct cpu_common_dbs_info *shared,
213 unsigned int sampling_rate) 211 unsigned int sampling_rate)
214{ 212{
215 if (policy_is_shared(cdbs->cur_policy)) { 213 if (policy_is_shared(shared->policy)) {
216 ktime_t time_now = ktime_get(); 214 ktime_t time_now = ktime_get();
217 s64 delta_us = ktime_us_delta(time_now, cdbs->time_stamp); 215 s64 delta_us = ktime_us_delta(time_now, shared->time_stamp);
218 216
219 /* Do nothing if we recently have sampled */ 217 /* Do nothing if we recently have sampled */
220 if (delta_us < (s64)(sampling_rate / 2)) 218 if (delta_us < (s64)(sampling_rate / 2))
221 return false; 219 return false;
222 else 220 else
223 cdbs->time_stamp = time_now; 221 shared->time_stamp = time_now;
224 } 222 }
225 223
226 return true; 224 return true;
227} 225}
228EXPORT_SYMBOL_GPL(need_load_eval); 226
227static void dbs_timer(struct work_struct *work)
228{
229 struct cpu_dbs_info *cdbs = container_of(work, struct cpu_dbs_info,
230 dwork.work);
231 struct cpu_common_dbs_info *shared = cdbs->shared;
232 struct cpufreq_policy *policy = shared->policy;
233 struct dbs_data *dbs_data = policy->governor_data;
234 unsigned int sampling_rate, delay;
235 bool modify_all = true;
236
237 mutex_lock(&shared->timer_mutex);
238
239 if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
240 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
241
242 sampling_rate = cs_tuners->sampling_rate;
243 } else {
244 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
245
246 sampling_rate = od_tuners->sampling_rate;
247 }
248
249 if (!need_load_eval(cdbs->shared, sampling_rate))
250 modify_all = false;
251
252 delay = dbs_data->cdata->gov_dbs_timer(cdbs, dbs_data, modify_all);
253 gov_queue_work(dbs_data, policy, delay, modify_all);
254
255 mutex_unlock(&shared->timer_mutex);
256}
229 257
230static void set_sampling_rate(struct dbs_data *dbs_data, 258static void set_sampling_rate(struct dbs_data *dbs_data,
231 unsigned int sampling_rate) 259 unsigned int sampling_rate)
@@ -239,6 +267,37 @@ static void set_sampling_rate(struct dbs_data *dbs_data,
239 } 267 }
240} 268}
241 269
270static int alloc_common_dbs_info(struct cpufreq_policy *policy,
271 struct common_dbs_data *cdata)
272{
273 struct cpu_common_dbs_info *shared;
274 int j;
275
276 /* Allocate memory for the common information for policy->cpus */
277 shared = kzalloc(sizeof(*shared), GFP_KERNEL);
278 if (!shared)
279 return -ENOMEM;
280
281 /* Set shared for all CPUs, online+offline */
282 for_each_cpu(j, policy->related_cpus)
283 cdata->get_cpu_cdbs(j)->shared = shared;
284
285 return 0;
286}
287
288static void free_common_dbs_info(struct cpufreq_policy *policy,
289 struct common_dbs_data *cdata)
290{
291 struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(policy->cpu);
292 struct cpu_common_dbs_info *shared = cdbs->shared;
293 int j;
294
295 for_each_cpu(j, policy->cpus)
296 cdata->get_cpu_cdbs(j)->shared = NULL;
297
298 kfree(shared);
299}
300
242static int cpufreq_governor_init(struct cpufreq_policy *policy, 301static int cpufreq_governor_init(struct cpufreq_policy *policy,
243 struct dbs_data *dbs_data, 302 struct dbs_data *dbs_data,
244 struct common_dbs_data *cdata) 303 struct common_dbs_data *cdata)
@@ -246,9 +305,18 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy,
246 unsigned int latency; 305 unsigned int latency;
247 int ret; 306 int ret;
248 307
308 /* State should be equivalent to EXIT */
309 if (policy->governor_data)
310 return -EBUSY;
311
249 if (dbs_data) { 312 if (dbs_data) {
250 if (WARN_ON(have_governor_per_policy())) 313 if (WARN_ON(have_governor_per_policy()))
251 return -EINVAL; 314 return -EINVAL;
315
316 ret = alloc_common_dbs_info(policy, cdata);
317 if (ret)
318 return ret;
319
252 dbs_data->usage_count++; 320 dbs_data->usage_count++;
253 policy->governor_data = dbs_data; 321 policy->governor_data = dbs_data;
254 return 0; 322 return 0;
@@ -258,12 +326,16 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy,
258 if (!dbs_data) 326 if (!dbs_data)
259 return -ENOMEM; 327 return -ENOMEM;
260 328
329 ret = alloc_common_dbs_info(policy, cdata);
330 if (ret)
331 goto free_dbs_data;
332
261 dbs_data->cdata = cdata; 333 dbs_data->cdata = cdata;
262 dbs_data->usage_count = 1; 334 dbs_data->usage_count = 1;
263 335
264 ret = cdata->init(dbs_data, !policy->governor->initialized); 336 ret = cdata->init(dbs_data, !policy->governor->initialized);
265 if (ret) 337 if (ret)
266 goto free_dbs_data; 338 goto free_common_dbs_info;
267 339
268 /* policy latency is in ns. Convert it to us first */ 340 /* policy latency is in ns. Convert it to us first */
269 latency = policy->cpuinfo.transition_latency / 1000; 341 latency = policy->cpuinfo.transition_latency / 1000;
@@ -300,15 +372,22 @@ put_kobj:
300 } 372 }
301cdata_exit: 373cdata_exit:
302 cdata->exit(dbs_data, !policy->governor->initialized); 374 cdata->exit(dbs_data, !policy->governor->initialized);
375free_common_dbs_info:
376 free_common_dbs_info(policy, cdata);
303free_dbs_data: 377free_dbs_data:
304 kfree(dbs_data); 378 kfree(dbs_data);
305 return ret; 379 return ret;
306} 380}
307 381
308static void cpufreq_governor_exit(struct cpufreq_policy *policy, 382static int cpufreq_governor_exit(struct cpufreq_policy *policy,
309 struct dbs_data *dbs_data) 383 struct dbs_data *dbs_data)
310{ 384{
311 struct common_dbs_data *cdata = dbs_data->cdata; 385 struct common_dbs_data *cdata = dbs_data->cdata;
386 struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(policy->cpu);
387
388 /* State should be equivalent to INIT */
389 if (!cdbs->shared || cdbs->shared->policy)
390 return -EBUSY;
312 391
313 policy->governor_data = NULL; 392 policy->governor_data = NULL;
314 if (!--dbs_data->usage_count) { 393 if (!--dbs_data->usage_count) {
@@ -323,6 +402,9 @@ static void cpufreq_governor_exit(struct cpufreq_policy *policy,
323 cdata->exit(dbs_data, policy->governor->initialized == 1); 402 cdata->exit(dbs_data, policy->governor->initialized == 1);
324 kfree(dbs_data); 403 kfree(dbs_data);
325 } 404 }
405
406 free_common_dbs_info(policy, cdata);
407 return 0;
326} 408}
327 409
328static int cpufreq_governor_start(struct cpufreq_policy *policy, 410static int cpufreq_governor_start(struct cpufreq_policy *policy,
@@ -330,12 +412,17 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy,
330{ 412{
331 struct common_dbs_data *cdata = dbs_data->cdata; 413 struct common_dbs_data *cdata = dbs_data->cdata;
332 unsigned int sampling_rate, ignore_nice, j, cpu = policy->cpu; 414 unsigned int sampling_rate, ignore_nice, j, cpu = policy->cpu;
333 struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu); 415 struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu);
416 struct cpu_common_dbs_info *shared = cdbs->shared;
334 int io_busy = 0; 417 int io_busy = 0;
335 418
336 if (!policy->cur) 419 if (!policy->cur)
337 return -EINVAL; 420 return -EINVAL;
338 421
422 /* State should be equivalent to INIT */
423 if (!shared || shared->policy)
424 return -EBUSY;
425
339 if (cdata->governor == GOV_CONSERVATIVE) { 426 if (cdata->governor == GOV_CONSERVATIVE) {
340 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; 427 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
341 428
@@ -349,12 +436,14 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy,
349 io_busy = od_tuners->io_is_busy; 436 io_busy = od_tuners->io_is_busy;
350 } 437 }
351 438
439 shared->policy = policy;
440 shared->time_stamp = ktime_get();
441 mutex_init(&shared->timer_mutex);
442
352 for_each_cpu(j, policy->cpus) { 443 for_each_cpu(j, policy->cpus) {
353 struct cpu_dbs_common_info *j_cdbs = cdata->get_cpu_cdbs(j); 444 struct cpu_dbs_info *j_cdbs = cdata->get_cpu_cdbs(j);
354 unsigned int prev_load; 445 unsigned int prev_load;
355 446
356 j_cdbs->cpu = j;
357 j_cdbs->cur_policy = policy;
358 j_cdbs->prev_cpu_idle = 447 j_cdbs->prev_cpu_idle =
359 get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy); 448 get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy);
360 449
@@ -366,8 +455,7 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy,
366 if (ignore_nice) 455 if (ignore_nice)
367 j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE]; 456 j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
368 457
369 mutex_init(&j_cdbs->timer_mutex); 458 INIT_DEFERRABLE_WORK(&j_cdbs->dwork, dbs_timer);
370 INIT_DEFERRABLE_WORK(&j_cdbs->work, cdata->gov_dbs_timer);
371 } 459 }
372 460
373 if (cdata->governor == GOV_CONSERVATIVE) { 461 if (cdata->governor == GOV_CONSERVATIVE) {
@@ -386,20 +474,24 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy,
386 od_ops->powersave_bias_init_cpu(cpu); 474 od_ops->powersave_bias_init_cpu(cpu);
387 } 475 }
388 476
389 /* Initiate timer time stamp */
390 cpu_cdbs->time_stamp = ktime_get();
391
392 gov_queue_work(dbs_data, policy, delay_for_sampling_rate(sampling_rate), 477 gov_queue_work(dbs_data, policy, delay_for_sampling_rate(sampling_rate),
393 true); 478 true);
394 return 0; 479 return 0;
395} 480}
396 481
397static void cpufreq_governor_stop(struct cpufreq_policy *policy, 482static int cpufreq_governor_stop(struct cpufreq_policy *policy,
398 struct dbs_data *dbs_data) 483 struct dbs_data *dbs_data)
399{ 484{
400 struct common_dbs_data *cdata = dbs_data->cdata; 485 struct common_dbs_data *cdata = dbs_data->cdata;
401 unsigned int cpu = policy->cpu; 486 unsigned int cpu = policy->cpu;
402 struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu); 487 struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu);
488 struct cpu_common_dbs_info *shared = cdbs->shared;
489
490 /* State should be equivalent to START */
491 if (!shared || !shared->policy)
492 return -EBUSY;
493
494 gov_cancel_work(dbs_data, policy);
403 495
404 if (cdata->governor == GOV_CONSERVATIVE) { 496 if (cdata->governor == GOV_CONSERVATIVE) {
405 struct cs_cpu_dbs_info_s *cs_dbs_info = 497 struct cs_cpu_dbs_info_s *cs_dbs_info =
@@ -408,38 +500,40 @@ static void cpufreq_governor_stop(struct cpufreq_policy *policy,
408 cs_dbs_info->enable = 0; 500 cs_dbs_info->enable = 0;
409 } 501 }
410 502
411 gov_cancel_work(dbs_data, policy); 503 shared->policy = NULL;
412 504 mutex_destroy(&shared->timer_mutex);
413 mutex_destroy(&cpu_cdbs->timer_mutex); 505 return 0;
414 cpu_cdbs->cur_policy = NULL;
415} 506}
416 507
417static void cpufreq_governor_limits(struct cpufreq_policy *policy, 508static int cpufreq_governor_limits(struct cpufreq_policy *policy,
418 struct dbs_data *dbs_data) 509 struct dbs_data *dbs_data)
419{ 510{
420 struct common_dbs_data *cdata = dbs_data->cdata; 511 struct common_dbs_data *cdata = dbs_data->cdata;
421 unsigned int cpu = policy->cpu; 512 unsigned int cpu = policy->cpu;
422 struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu); 513 struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu);
423 514
424 if (!cpu_cdbs->cur_policy) 515 /* State should be equivalent to START */
425 return; 516 if (!cdbs->shared || !cdbs->shared->policy)
517 return -EBUSY;
426 518
427 mutex_lock(&cpu_cdbs->timer_mutex); 519 mutex_lock(&cdbs->shared->timer_mutex);
428 if (policy->max < cpu_cdbs->cur_policy->cur) 520 if (policy->max < cdbs->shared->policy->cur)
429 __cpufreq_driver_target(cpu_cdbs->cur_policy, policy->max, 521 __cpufreq_driver_target(cdbs->shared->policy, policy->max,
430 CPUFREQ_RELATION_H); 522 CPUFREQ_RELATION_H);
431 else if (policy->min > cpu_cdbs->cur_policy->cur) 523 else if (policy->min > cdbs->shared->policy->cur)
432 __cpufreq_driver_target(cpu_cdbs->cur_policy, policy->min, 524 __cpufreq_driver_target(cdbs->shared->policy, policy->min,
433 CPUFREQ_RELATION_L); 525 CPUFREQ_RELATION_L);
434 dbs_check_cpu(dbs_data, cpu); 526 dbs_check_cpu(dbs_data, cpu);
435 mutex_unlock(&cpu_cdbs->timer_mutex); 527 mutex_unlock(&cdbs->shared->timer_mutex);
528
529 return 0;
436} 530}
437 531
438int cpufreq_governor_dbs(struct cpufreq_policy *policy, 532int cpufreq_governor_dbs(struct cpufreq_policy *policy,
439 struct common_dbs_data *cdata, unsigned int event) 533 struct common_dbs_data *cdata, unsigned int event)
440{ 534{
441 struct dbs_data *dbs_data; 535 struct dbs_data *dbs_data;
442 int ret = 0; 536 int ret;
443 537
444 /* Lock governor to block concurrent initialization of governor */ 538 /* Lock governor to block concurrent initialization of governor */
445 mutex_lock(&cdata->mutex); 539 mutex_lock(&cdata->mutex);
@@ -449,7 +543,7 @@ int cpufreq_governor_dbs(struct cpufreq_policy *policy,
449 else 543 else
450 dbs_data = cdata->gdbs_data; 544 dbs_data = cdata->gdbs_data;
451 545
452 if (WARN_ON(!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT))) { 546 if (!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT)) {
453 ret = -EINVAL; 547 ret = -EINVAL;
454 goto unlock; 548 goto unlock;
455 } 549 }
@@ -459,17 +553,19 @@ int cpufreq_governor_dbs(struct cpufreq_policy *policy,
459 ret = cpufreq_governor_init(policy, dbs_data, cdata); 553 ret = cpufreq_governor_init(policy, dbs_data, cdata);
460 break; 554 break;
461 case CPUFREQ_GOV_POLICY_EXIT: 555 case CPUFREQ_GOV_POLICY_EXIT:
462 cpufreq_governor_exit(policy, dbs_data); 556 ret = cpufreq_governor_exit(policy, dbs_data);
463 break; 557 break;
464 case CPUFREQ_GOV_START: 558 case CPUFREQ_GOV_START:
465 ret = cpufreq_governor_start(policy, dbs_data); 559 ret = cpufreq_governor_start(policy, dbs_data);
466 break; 560 break;
467 case CPUFREQ_GOV_STOP: 561 case CPUFREQ_GOV_STOP:
468 cpufreq_governor_stop(policy, dbs_data); 562 ret = cpufreq_governor_stop(policy, dbs_data);
469 break; 563 break;
470 case CPUFREQ_GOV_LIMITS: 564 case CPUFREQ_GOV_LIMITS:
471 cpufreq_governor_limits(policy, dbs_data); 565 ret = cpufreq_governor_limits(policy, dbs_data);
472 break; 566 break;
567 default:
568 ret = -EINVAL;
473 } 569 }
474 570
475unlock: 571unlock:
diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h
index 34736f5e869d..50f171796632 100644
--- a/drivers/cpufreq/cpufreq_governor.h
+++ b/drivers/cpufreq/cpufreq_governor.h
@@ -109,7 +109,7 @@ store_one(_gov, file_name)
109 109
110/* create helper routines */ 110/* create helper routines */
111#define define_get_cpu_dbs_routines(_dbs_info) \ 111#define define_get_cpu_dbs_routines(_dbs_info) \
112static struct cpu_dbs_common_info *get_cpu_cdbs(int cpu) \ 112static struct cpu_dbs_info *get_cpu_cdbs(int cpu) \
113{ \ 113{ \
114 return &per_cpu(_dbs_info, cpu).cdbs; \ 114 return &per_cpu(_dbs_info, cpu).cdbs; \
115} \ 115} \
@@ -128,9 +128,20 @@ static void *get_cpu_dbs_info_s(int cpu) \
128 * cs_*: Conservative governor 128 * cs_*: Conservative governor
129 */ 129 */
130 130
131/* Common to all CPUs of a policy */
132struct cpu_common_dbs_info {
133 struct cpufreq_policy *policy;
134 /*
135 * percpu mutex that serializes governor limit change with dbs_timer
136 * invocation. We do not want dbs_timer to run when user is changing
137 * the governor or limits.
138 */
139 struct mutex timer_mutex;
140 ktime_t time_stamp;
141};
142
131/* Per cpu structures */ 143/* Per cpu structures */
132struct cpu_dbs_common_info { 144struct cpu_dbs_info {
133 int cpu;
134 u64 prev_cpu_idle; 145 u64 prev_cpu_idle;
135 u64 prev_cpu_wall; 146 u64 prev_cpu_wall;
136 u64 prev_cpu_nice; 147 u64 prev_cpu_nice;
@@ -141,19 +152,12 @@ struct cpu_dbs_common_info {
141 * wake-up from idle. 152 * wake-up from idle.
142 */ 153 */
143 unsigned int prev_load; 154 unsigned int prev_load;
144 struct cpufreq_policy *cur_policy; 155 struct delayed_work dwork;
145 struct delayed_work work; 156 struct cpu_common_dbs_info *shared;
146 /*
147 * percpu mutex that serializes governor limit change with gov_dbs_timer
148 * invocation. We do not want gov_dbs_timer to run when user is changing
149 * the governor or limits.
150 */
151 struct mutex timer_mutex;
152 ktime_t time_stamp;
153}; 157};
154 158
155struct od_cpu_dbs_info_s { 159struct od_cpu_dbs_info_s {
156 struct cpu_dbs_common_info cdbs; 160 struct cpu_dbs_info cdbs;
157 struct cpufreq_frequency_table *freq_table; 161 struct cpufreq_frequency_table *freq_table;
158 unsigned int freq_lo; 162 unsigned int freq_lo;
159 unsigned int freq_lo_jiffies; 163 unsigned int freq_lo_jiffies;
@@ -163,7 +167,7 @@ struct od_cpu_dbs_info_s {
163}; 167};
164 168
165struct cs_cpu_dbs_info_s { 169struct cs_cpu_dbs_info_s {
166 struct cpu_dbs_common_info cdbs; 170 struct cpu_dbs_info cdbs;
167 unsigned int down_skip; 171 unsigned int down_skip;
168 unsigned int requested_freq; 172 unsigned int requested_freq;
169 unsigned int enable:1; 173 unsigned int enable:1;
@@ -204,9 +208,11 @@ struct common_dbs_data {
204 */ 208 */
205 struct dbs_data *gdbs_data; 209 struct dbs_data *gdbs_data;
206 210
207 struct cpu_dbs_common_info *(*get_cpu_cdbs)(int cpu); 211 struct cpu_dbs_info *(*get_cpu_cdbs)(int cpu);
208 void *(*get_cpu_dbs_info_s)(int cpu); 212 void *(*get_cpu_dbs_info_s)(int cpu);
209 void (*gov_dbs_timer)(struct work_struct *work); 213 unsigned int (*gov_dbs_timer)(struct cpu_dbs_info *cdbs,
214 struct dbs_data *dbs_data,
215 bool modify_all);
210 void (*gov_check_cpu)(int cpu, unsigned int load); 216 void (*gov_check_cpu)(int cpu, unsigned int load);
211 int (*init)(struct dbs_data *dbs_data, bool notify); 217 int (*init)(struct dbs_data *dbs_data, bool notify);
212 void (*exit)(struct dbs_data *dbs_data, bool notify); 218 void (*exit)(struct dbs_data *dbs_data, bool notify);
@@ -265,8 +271,6 @@ static ssize_t show_sampling_rate_min_gov_pol \
265extern struct mutex cpufreq_governor_lock; 271extern struct mutex cpufreq_governor_lock;
266 272
267void dbs_check_cpu(struct dbs_data *dbs_data, int cpu); 273void dbs_check_cpu(struct dbs_data *dbs_data, int cpu);
268bool need_load_eval(struct cpu_dbs_common_info *cdbs,
269 unsigned int sampling_rate);
270int cpufreq_governor_dbs(struct cpufreq_policy *policy, 274int cpufreq_governor_dbs(struct cpufreq_policy *policy,
271 struct common_dbs_data *cdata, unsigned int event); 275 struct common_dbs_data *cdata, unsigned int event);
272void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy, 276void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy,
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
index 3c1e10f2304c..1fa9088c84a8 100644
--- a/drivers/cpufreq/cpufreq_ondemand.c
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -155,7 +155,7 @@ static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq)
155static void od_check_cpu(int cpu, unsigned int load) 155static void od_check_cpu(int cpu, unsigned int load)
156{ 156{
157 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); 157 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
158 struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; 158 struct cpufreq_policy *policy = dbs_info->cdbs.shared->policy;
159 struct dbs_data *dbs_data = policy->governor_data; 159 struct dbs_data *dbs_data = policy->governor_data;
160 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 160 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
161 161
@@ -191,46 +191,40 @@ static void od_check_cpu(int cpu, unsigned int load)
191 } 191 }
192} 192}
193 193
194static void od_dbs_timer(struct work_struct *work) 194static unsigned int od_dbs_timer(struct cpu_dbs_info *cdbs,
195 struct dbs_data *dbs_data, bool modify_all)
195{ 196{
196 struct od_cpu_dbs_info_s *dbs_info = 197 struct cpufreq_policy *policy = cdbs->shared->policy;
197 container_of(work, struct od_cpu_dbs_info_s, cdbs.work.work); 198 unsigned int cpu = policy->cpu;
198 unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; 199 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
199 struct od_cpu_dbs_info_s *core_dbs_info = &per_cpu(od_cpu_dbs_info,
200 cpu); 200 cpu);
201 struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data;
202 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 201 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
203 int delay = 0, sample_type = core_dbs_info->sample_type; 202 int delay = 0, sample_type = dbs_info->sample_type;
204 bool modify_all = true;
205 203
206 mutex_lock(&core_dbs_info->cdbs.timer_mutex); 204 if (!modify_all)
207 if (!need_load_eval(&core_dbs_info->cdbs, od_tuners->sampling_rate)) {
208 modify_all = false;
209 goto max_delay; 205 goto max_delay;
210 }
211 206
212 /* Common NORMAL_SAMPLE setup */ 207 /* Common NORMAL_SAMPLE setup */
213 core_dbs_info->sample_type = OD_NORMAL_SAMPLE; 208 dbs_info->sample_type = OD_NORMAL_SAMPLE;
214 if (sample_type == OD_SUB_SAMPLE) { 209 if (sample_type == OD_SUB_SAMPLE) {
215 delay = core_dbs_info->freq_lo_jiffies; 210 delay = dbs_info->freq_lo_jiffies;
216 __cpufreq_driver_target(core_dbs_info->cdbs.cur_policy, 211 __cpufreq_driver_target(policy, dbs_info->freq_lo,
217 core_dbs_info->freq_lo, CPUFREQ_RELATION_H); 212 CPUFREQ_RELATION_H);
218 } else { 213 } else {
219 dbs_check_cpu(dbs_data, cpu); 214 dbs_check_cpu(dbs_data, cpu);
220 if (core_dbs_info->freq_lo) { 215 if (dbs_info->freq_lo) {
221 /* Setup timer for SUB_SAMPLE */ 216 /* Setup timer for SUB_SAMPLE */
222 core_dbs_info->sample_type = OD_SUB_SAMPLE; 217 dbs_info->sample_type = OD_SUB_SAMPLE;
223 delay = core_dbs_info->freq_hi_jiffies; 218 delay = dbs_info->freq_hi_jiffies;
224 } 219 }
225 } 220 }
226 221
227max_delay: 222max_delay:
228 if (!delay) 223 if (!delay)
229 delay = delay_for_sampling_rate(od_tuners->sampling_rate 224 delay = delay_for_sampling_rate(od_tuners->sampling_rate
230 * core_dbs_info->rate_mult); 225 * dbs_info->rate_mult);
231 226
232 gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, delay, modify_all); 227 return delay;
233 mutex_unlock(&core_dbs_info->cdbs.timer_mutex);
234} 228}
235 229
236/************************** sysfs interface ************************/ 230/************************** sysfs interface ************************/
@@ -273,27 +267,27 @@ static void update_sampling_rate(struct dbs_data *dbs_data,
273 dbs_info = &per_cpu(od_cpu_dbs_info, cpu); 267 dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
274 cpufreq_cpu_put(policy); 268 cpufreq_cpu_put(policy);
275 269
276 mutex_lock(&dbs_info->cdbs.timer_mutex); 270 mutex_lock(&dbs_info->cdbs.shared->timer_mutex);
277 271
278 if (!delayed_work_pending(&dbs_info->cdbs.work)) { 272 if (!delayed_work_pending(&dbs_info->cdbs.dwork)) {
279 mutex_unlock(&dbs_info->cdbs.timer_mutex); 273 mutex_unlock(&dbs_info->cdbs.shared->timer_mutex);
280 continue; 274 continue;
281 } 275 }
282 276
283 next_sampling = jiffies + usecs_to_jiffies(new_rate); 277 next_sampling = jiffies + usecs_to_jiffies(new_rate);
284 appointed_at = dbs_info->cdbs.work.timer.expires; 278 appointed_at = dbs_info->cdbs.dwork.timer.expires;
285 279
286 if (time_before(next_sampling, appointed_at)) { 280 if (time_before(next_sampling, appointed_at)) {
287 281
288 mutex_unlock(&dbs_info->cdbs.timer_mutex); 282 mutex_unlock(&dbs_info->cdbs.shared->timer_mutex);
289 cancel_delayed_work_sync(&dbs_info->cdbs.work); 283 cancel_delayed_work_sync(&dbs_info->cdbs.dwork);
290 mutex_lock(&dbs_info->cdbs.timer_mutex); 284 mutex_lock(&dbs_info->cdbs.shared->timer_mutex);
291 285
292 gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, 286 gov_queue_work(dbs_data, policy,
293 usecs_to_jiffies(new_rate), true); 287 usecs_to_jiffies(new_rate), true);
294 288
295 } 289 }
296 mutex_unlock(&dbs_info->cdbs.timer_mutex); 290 mutex_unlock(&dbs_info->cdbs.shared->timer_mutex);
297 } 291 }
298} 292}
299 293
@@ -556,13 +550,16 @@ static void od_set_powersave_bias(unsigned int powersave_bias)
556 550
557 get_online_cpus(); 551 get_online_cpus();
558 for_each_online_cpu(cpu) { 552 for_each_online_cpu(cpu) {
553 struct cpu_common_dbs_info *shared;
554
559 if (cpumask_test_cpu(cpu, &done)) 555 if (cpumask_test_cpu(cpu, &done))
560 continue; 556 continue;
561 557
562 policy = per_cpu(od_cpu_dbs_info, cpu).cdbs.cur_policy; 558 shared = per_cpu(od_cpu_dbs_info, cpu).cdbs.shared;
563 if (!policy) 559 if (!shared)
564 continue; 560 continue;
565 561
562 policy = shared->policy;
566 cpumask_or(&done, &done, policy->cpus); 563 cpumask_or(&done, &done, policy->cpus);
567 564
568 if (policy->governor != &cpufreq_gov_ondemand) 565 if (policy->governor != &cpufreq_gov_ondemand)
diff --git a/drivers/cpufreq/e_powersaver.c b/drivers/cpufreq/e_powersaver.c
index a0d2a423cea9..4085244c8a67 100644
--- a/drivers/cpufreq/e_powersaver.c
+++ b/drivers/cpufreq/e_powersaver.c
@@ -78,7 +78,7 @@ static int eps_acpi_init(void)
78static int eps_acpi_exit(struct cpufreq_policy *policy) 78static int eps_acpi_exit(struct cpufreq_policy *policy)
79{ 79{
80 if (eps_acpi_cpu_perf) { 80 if (eps_acpi_cpu_perf) {
81 acpi_processor_unregister_performance(eps_acpi_cpu_perf, 0); 81 acpi_processor_unregister_performance(0);
82 free_cpumask_var(eps_acpi_cpu_perf->shared_cpu_map); 82 free_cpumask_var(eps_acpi_cpu_perf->shared_cpu_map);
83 kfree(eps_acpi_cpu_perf); 83 kfree(eps_acpi_cpu_perf);
84 eps_acpi_cpu_perf = NULL; 84 eps_acpi_cpu_perf = NULL;
diff --git a/drivers/cpufreq/ia64-acpi-cpufreq.c b/drivers/cpufreq/ia64-acpi-cpufreq.c
index c30aaa6a54e8..0202429f1c5b 100644
--- a/drivers/cpufreq/ia64-acpi-cpufreq.c
+++ b/drivers/cpufreq/ia64-acpi-cpufreq.c
@@ -29,7 +29,6 @@ MODULE_LICENSE("GPL");
29 29
30struct cpufreq_acpi_io { 30struct cpufreq_acpi_io {
31 struct acpi_processor_performance acpi_data; 31 struct acpi_processor_performance acpi_data;
32 struct cpufreq_frequency_table *freq_table;
33 unsigned int resume; 32 unsigned int resume;
34}; 33};
35 34
@@ -221,6 +220,7 @@ acpi_cpufreq_cpu_init (
221 unsigned int cpu = policy->cpu; 220 unsigned int cpu = policy->cpu;
222 struct cpufreq_acpi_io *data; 221 struct cpufreq_acpi_io *data;
223 unsigned int result = 0; 222 unsigned int result = 0;
223 struct cpufreq_frequency_table *freq_table;
224 224
225 pr_debug("acpi_cpufreq_cpu_init\n"); 225 pr_debug("acpi_cpufreq_cpu_init\n");
226 226
@@ -254,10 +254,10 @@ acpi_cpufreq_cpu_init (
254 } 254 }
255 255
256 /* alloc freq_table */ 256 /* alloc freq_table */
257 data->freq_table = kzalloc(sizeof(*data->freq_table) * 257 freq_table = kzalloc(sizeof(*freq_table) *
258 (data->acpi_data.state_count + 1), 258 (data->acpi_data.state_count + 1),
259 GFP_KERNEL); 259 GFP_KERNEL);
260 if (!data->freq_table) { 260 if (!freq_table) {
261 result = -ENOMEM; 261 result = -ENOMEM;
262 goto err_unreg; 262 goto err_unreg;
263 } 263 }
@@ -276,14 +276,14 @@ acpi_cpufreq_cpu_init (
276 for (i = 0; i <= data->acpi_data.state_count; i++) 276 for (i = 0; i <= data->acpi_data.state_count; i++)
277 { 277 {
278 if (i < data->acpi_data.state_count) { 278 if (i < data->acpi_data.state_count) {
279 data->freq_table[i].frequency = 279 freq_table[i].frequency =
280 data->acpi_data.states[i].core_frequency * 1000; 280 data->acpi_data.states[i].core_frequency * 1000;
281 } else { 281 } else {
282 data->freq_table[i].frequency = CPUFREQ_TABLE_END; 282 freq_table[i].frequency = CPUFREQ_TABLE_END;
283 } 283 }
284 } 284 }
285 285
286 result = cpufreq_table_validate_and_show(policy, data->freq_table); 286 result = cpufreq_table_validate_and_show(policy, freq_table);
287 if (result) { 287 if (result) {
288 goto err_freqfree; 288 goto err_freqfree;
289 } 289 }
@@ -311,9 +311,9 @@ acpi_cpufreq_cpu_init (
311 return (result); 311 return (result);
312 312
313 err_freqfree: 313 err_freqfree:
314 kfree(data->freq_table); 314 kfree(freq_table);
315 err_unreg: 315 err_unreg:
316 acpi_processor_unregister_performance(&data->acpi_data, cpu); 316 acpi_processor_unregister_performance(cpu);
317 err_free: 317 err_free:
318 kfree(data); 318 kfree(data);
319 acpi_io_data[cpu] = NULL; 319 acpi_io_data[cpu] = NULL;
@@ -332,8 +332,8 @@ acpi_cpufreq_cpu_exit (
332 332
333 if (data) { 333 if (data) {
334 acpi_io_data[policy->cpu] = NULL; 334 acpi_io_data[policy->cpu] = NULL;
335 acpi_processor_unregister_performance(&data->acpi_data, 335 acpi_processor_unregister_performance(policy->cpu);
336 policy->cpu); 336 kfree(policy->freq_table);
337 kfree(data); 337 kfree(data);
338 } 338 }
339 339
diff --git a/drivers/cpufreq/integrator-cpufreq.c b/drivers/cpufreq/integrator-cpufreq.c
index 129e266f7621..2faa4216bf2a 100644
--- a/drivers/cpufreq/integrator-cpufreq.c
+++ b/drivers/cpufreq/integrator-cpufreq.c
@@ -98,11 +98,10 @@ static int integrator_set_target(struct cpufreq_policy *policy,
98 /* get current setting */ 98 /* get current setting */
99 cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET); 99 cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET);
100 100
101 if (machine_is_integrator()) { 101 if (machine_is_integrator())
102 vco.s = (cm_osc >> 8) & 7; 102 vco.s = (cm_osc >> 8) & 7;
103 } else if (machine_is_cintegrator()) { 103 else if (machine_is_cintegrator())
104 vco.s = 1; 104 vco.s = 1;
105 }
106 vco.v = cm_osc & 255; 105 vco.v = cm_osc & 255;
107 vco.r = 22; 106 vco.r = 22;
108 freqs.old = icst_hz(&cclk_params, vco) / 1000; 107 freqs.old = icst_hz(&cclk_params, vco) / 1000;
@@ -163,11 +162,10 @@ static unsigned int integrator_get(unsigned int cpu)
163 /* detect memory etc. */ 162 /* detect memory etc. */
164 cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET); 163 cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET);
165 164
166 if (machine_is_integrator()) { 165 if (machine_is_integrator())
167 vco.s = (cm_osc >> 8) & 7; 166 vco.s = (cm_osc >> 8) & 7;
168 } else { 167 else
169 vco.s = 1; 168 vco.s = 1;
170 }
171 vco.v = cm_osc & 255; 169 vco.v = cm_osc & 255;
172 vco.r = 22; 170 vco.r = 22;
173 171
@@ -203,7 +201,7 @@ static int __init integrator_cpufreq_probe(struct platform_device *pdev)
203 struct resource *res; 201 struct resource *res;
204 202
205 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 203 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
206 if (!res) 204 if (!res)
207 return -ENODEV; 205 return -ENODEV;
208 206
209 cm_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); 207 cm_base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
@@ -234,6 +232,6 @@ static struct platform_driver integrator_cpufreq_driver = {
234module_platform_driver_probe(integrator_cpufreq_driver, 232module_platform_driver_probe(integrator_cpufreq_driver,
235 integrator_cpufreq_probe); 233 integrator_cpufreq_probe);
236 234
237MODULE_AUTHOR ("Russell M. King"); 235MODULE_AUTHOR("Russell M. King");
238MODULE_DESCRIPTION ("cpufreq driver for ARM Integrator CPUs"); 236MODULE_DESCRIPTION("cpufreq driver for ARM Integrator CPUs");
239MODULE_LICENSE ("GPL"); 237MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index fcb929ec5304..31d0548638e8 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -484,12 +484,11 @@ static void __init intel_pstate_sysfs_expose_params(void)
484} 484}
485/************************** sysfs end ************************/ 485/************************** sysfs end ************************/
486 486
487static void intel_pstate_hwp_enable(void) 487static void intel_pstate_hwp_enable(struct cpudata *cpudata)
488{ 488{
489 hwp_active++;
490 pr_info("intel_pstate: HWP enabled\n"); 489 pr_info("intel_pstate: HWP enabled\n");
491 490
492 wrmsrl( MSR_PM_ENABLE, 0x1); 491 wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
493} 492}
494 493
495static int byt_get_min_pstate(void) 494static int byt_get_min_pstate(void)
@@ -522,7 +521,7 @@ static void byt_set_pstate(struct cpudata *cpudata, int pstate)
522 int32_t vid_fp; 521 int32_t vid_fp;
523 u32 vid; 522 u32 vid;
524 523
525 val = pstate << 8; 524 val = (u64)pstate << 8;
526 if (limits.no_turbo && !limits.turbo_disabled) 525 if (limits.no_turbo && !limits.turbo_disabled)
527 val |= (u64)1 << 32; 526 val |= (u64)1 << 32;
528 527
@@ -611,7 +610,7 @@ static void core_set_pstate(struct cpudata *cpudata, int pstate)
611{ 610{
612 u64 val; 611 u64 val;
613 612
614 val = pstate << 8; 613 val = (u64)pstate << 8;
615 if (limits.no_turbo && !limits.turbo_disabled) 614 if (limits.no_turbo && !limits.turbo_disabled)
616 val |= (u64)1 << 32; 615 val |= (u64)1 << 32;
617 616
@@ -909,6 +908,7 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
909 ICPU(0x4c, byt_params), 908 ICPU(0x4c, byt_params),
910 ICPU(0x4e, core_params), 909 ICPU(0x4e, core_params),
911 ICPU(0x4f, core_params), 910 ICPU(0x4f, core_params),
911 ICPU(0x5e, core_params),
912 ICPU(0x56, core_params), 912 ICPU(0x56, core_params),
913 ICPU(0x57, knl_params), 913 ICPU(0x57, knl_params),
914 {} 914 {}
@@ -933,6 +933,10 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
933 cpu = all_cpu_data[cpunum]; 933 cpu = all_cpu_data[cpunum];
934 934
935 cpu->cpu = cpunum; 935 cpu->cpu = cpunum;
936
937 if (hwp_active)
938 intel_pstate_hwp_enable(cpu);
939
936 intel_pstate_get_cpu_pstates(cpu); 940 intel_pstate_get_cpu_pstates(cpu);
937 941
938 init_timer_deferrable(&cpu->timer); 942 init_timer_deferrable(&cpu->timer);
@@ -1170,6 +1174,10 @@ static struct hw_vendor_info vendor_info[] = {
1170 {1, "ORACLE", "X4270M3 ", PPC}, 1174 {1, "ORACLE", "X4270M3 ", PPC},
1171 {1, "ORACLE", "X4270M2 ", PPC}, 1175 {1, "ORACLE", "X4270M2 ", PPC},
1172 {1, "ORACLE", "X4170M2 ", PPC}, 1176 {1, "ORACLE", "X4170M2 ", PPC},
1177 {1, "ORACLE", "X4170 M3", PPC},
1178 {1, "ORACLE", "X4275 M3", PPC},
1179 {1, "ORACLE", "X6-2 ", PPC},
1180 {1, "ORACLE", "Sudbury ", PPC},
1173 {0, "", ""}, 1181 {0, "", ""},
1174}; 1182};
1175 1183
@@ -1246,7 +1254,7 @@ static int __init intel_pstate_init(void)
1246 return -ENOMEM; 1254 return -ENOMEM;
1247 1255
1248 if (static_cpu_has_safe(X86_FEATURE_HWP) && !no_hwp) 1256 if (static_cpu_has_safe(X86_FEATURE_HWP) && !no_hwp)
1249 intel_pstate_hwp_enable(); 1257 hwp_active++;
1250 1258
1251 if (!hwp_active && hwp_only) 1259 if (!hwp_active && hwp_only)
1252 goto out; 1260 goto out;
diff --git a/drivers/cpufreq/mt8173-cpufreq.c b/drivers/cpufreq/mt8173-cpufreq.c
new file mode 100644
index 000000000000..49caed293a3b
--- /dev/null
+++ b/drivers/cpufreq/mt8173-cpufreq.c
@@ -0,0 +1,527 @@
1/*
2 * Copyright (c) 2015 Linaro Ltd.
3 * Author: Pi-Cheng Chen <pi-cheng.chen@linaro.org>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 */
14
15#include <linux/clk.h>
16#include <linux/cpu.h>
17#include <linux/cpu_cooling.h>
18#include <linux/cpufreq.h>
19#include <linux/cpumask.h>
20#include <linux/of.h>
21#include <linux/platform_device.h>
22#include <linux/pm_opp.h>
23#include <linux/regulator/consumer.h>
24#include <linux/slab.h>
25#include <linux/thermal.h>
26
27#define MIN_VOLT_SHIFT (100000)
28#define MAX_VOLT_SHIFT (200000)
29#define MAX_VOLT_LIMIT (1150000)
30#define VOLT_TOL (10000)
31
32/*
33 * The struct mtk_cpu_dvfs_info holds necessary information for doing CPU DVFS
34 * on each CPU power/clock domain of Mediatek SoCs. Each CPU cluster in
35 * Mediatek SoCs has two voltage inputs, Vproc and Vsram. In some cases the two
36 * voltage inputs need to be controlled under a hardware limitation:
37 * 100mV < Vsram - Vproc < 200mV
38 *
39 * When scaling the clock frequency of a CPU clock domain, the clock source
40 * needs to be switched to another stable PLL clock temporarily until
41 * the original PLL becomes stable at target frequency.
42 */
43struct mtk_cpu_dvfs_info {
44 struct device *cpu_dev;
45 struct regulator *proc_reg;
46 struct regulator *sram_reg;
47 struct clk *cpu_clk;
48 struct clk *inter_clk;
49 struct thermal_cooling_device *cdev;
50 int intermediate_voltage;
51 bool need_voltage_tracking;
52};
53
54static int mtk_cpufreq_voltage_tracking(struct mtk_cpu_dvfs_info *info,
55 int new_vproc)
56{
57 struct regulator *proc_reg = info->proc_reg;
58 struct regulator *sram_reg = info->sram_reg;
59 int old_vproc, old_vsram, new_vsram, vsram, vproc, ret;
60
61 old_vproc = regulator_get_voltage(proc_reg);
62 old_vsram = regulator_get_voltage(sram_reg);
63 /* Vsram should not exceed the maximum allowed voltage of SoC. */
64 new_vsram = min(new_vproc + MIN_VOLT_SHIFT, MAX_VOLT_LIMIT);
65
66 if (old_vproc < new_vproc) {
67 /*
68 * When scaling up voltages, Vsram and Vproc scale up step
69 * by step. At each step, set Vsram to (Vproc + 200mV) first,
70 * then set Vproc to (Vsram - 100mV).
71 * Keep doing it until Vsram and Vproc hit target voltages.
72 */
73 do {
74 old_vsram = regulator_get_voltage(sram_reg);
75 old_vproc = regulator_get_voltage(proc_reg);
76
77 vsram = min(new_vsram, old_vproc + MAX_VOLT_SHIFT);
78
79 if (vsram + VOLT_TOL >= MAX_VOLT_LIMIT) {
80 vsram = MAX_VOLT_LIMIT;
81
82 /*
83 * If the target Vsram hits the maximum voltage,
84 * try to set the exact voltage value first.
85 */
86 ret = regulator_set_voltage(sram_reg, vsram,
87 vsram);
88 if (ret)
89 ret = regulator_set_voltage(sram_reg,
90 vsram - VOLT_TOL,
91 vsram);
92
93 vproc = new_vproc;
94 } else {
95 ret = regulator_set_voltage(sram_reg, vsram,
96 vsram + VOLT_TOL);
97
98 vproc = vsram - MIN_VOLT_SHIFT;
99 }
100 if (ret)
101 return ret;
102
103 ret = regulator_set_voltage(proc_reg, vproc,
104 vproc + VOLT_TOL);
105 if (ret) {
106 regulator_set_voltage(sram_reg, old_vsram,
107 old_vsram);
108 return ret;
109 }
110 } while (vproc < new_vproc || vsram < new_vsram);
111 } else if (old_vproc > new_vproc) {
112 /*
113 * When scaling down voltages, Vsram and Vproc scale down step
114 * by step. At each step, set Vproc to (Vsram - 200mV) first,
115 * then set Vproc to (Vproc + 100mV).
116 * Keep doing it until Vsram and Vproc hit target voltages.
117 */
118 do {
119 old_vproc = regulator_get_voltage(proc_reg);
120 old_vsram = regulator_get_voltage(sram_reg);
121
122 vproc = max(new_vproc, old_vsram - MAX_VOLT_SHIFT);
123 ret = regulator_set_voltage(proc_reg, vproc,
124 vproc + VOLT_TOL);
125 if (ret)
126 return ret;
127
128 if (vproc == new_vproc)
129 vsram = new_vsram;
130 else
131 vsram = max(new_vsram, vproc + MIN_VOLT_SHIFT);
132
133 if (vsram + VOLT_TOL >= MAX_VOLT_LIMIT) {
134 vsram = MAX_VOLT_LIMIT;
135
136 /*
137 * If the target Vsram hits the maximum voltage,
138 * try to set the exact voltage value first.
139 */
140 ret = regulator_set_voltage(sram_reg, vsram,
141 vsram);
142 if (ret)
143 ret = regulator_set_voltage(sram_reg,
144 vsram - VOLT_TOL,
145 vsram);
146 } else {
147 ret = regulator_set_voltage(sram_reg, vsram,
148 vsram + VOLT_TOL);
149 }
150
151 if (ret) {
152 regulator_set_voltage(proc_reg, old_vproc,
153 old_vproc);
154 return ret;
155 }
156 } while (vproc > new_vproc + VOLT_TOL ||
157 vsram > new_vsram + VOLT_TOL);
158 }
159
160 return 0;
161}
162
163static int mtk_cpufreq_set_voltage(struct mtk_cpu_dvfs_info *info, int vproc)
164{
165 if (info->need_voltage_tracking)
166 return mtk_cpufreq_voltage_tracking(info, vproc);
167 else
168 return regulator_set_voltage(info->proc_reg, vproc,
169 vproc + VOLT_TOL);
170}
171
172static int mtk_cpufreq_set_target(struct cpufreq_policy *policy,
173 unsigned int index)
174{
175 struct cpufreq_frequency_table *freq_table = policy->freq_table;
176 struct clk *cpu_clk = policy->clk;
177 struct clk *armpll = clk_get_parent(cpu_clk);
178 struct mtk_cpu_dvfs_info *info = policy->driver_data;
179 struct device *cpu_dev = info->cpu_dev;
180 struct dev_pm_opp *opp;
181 long freq_hz, old_freq_hz;
182 int vproc, old_vproc, inter_vproc, target_vproc, ret;
183
184 inter_vproc = info->intermediate_voltage;
185
186 old_freq_hz = clk_get_rate(cpu_clk);
187 old_vproc = regulator_get_voltage(info->proc_reg);
188
189 freq_hz = freq_table[index].frequency * 1000;
190
191 rcu_read_lock();
192 opp = dev_pm_opp_find_freq_ceil(cpu_dev, &freq_hz);
193 if (IS_ERR(opp)) {
194 rcu_read_unlock();
195 pr_err("cpu%d: failed to find OPP for %ld\n",
196 policy->cpu, freq_hz);
197 return PTR_ERR(opp);
198 }
199 vproc = dev_pm_opp_get_voltage(opp);
200 rcu_read_unlock();
201
202 /*
203 * If the new voltage or the intermediate voltage is higher than the
204 * current voltage, scale up voltage first.
205 */
206 target_vproc = (inter_vproc > vproc) ? inter_vproc : vproc;
207 if (old_vproc < target_vproc) {
208 ret = mtk_cpufreq_set_voltage(info, target_vproc);
209 if (ret) {
210 pr_err("cpu%d: failed to scale up voltage!\n",
211 policy->cpu);
212 mtk_cpufreq_set_voltage(info, old_vproc);
213 return ret;
214 }
215 }
216
217 /* Reparent the CPU clock to intermediate clock. */
218 ret = clk_set_parent(cpu_clk, info->inter_clk);
219 if (ret) {
220 pr_err("cpu%d: failed to re-parent cpu clock!\n",
221 policy->cpu);
222 mtk_cpufreq_set_voltage(info, old_vproc);
223 WARN_ON(1);
224 return ret;
225 }
226
227 /* Set the original PLL to target rate. */
228 ret = clk_set_rate(armpll, freq_hz);
229 if (ret) {
230 pr_err("cpu%d: failed to scale cpu clock rate!\n",
231 policy->cpu);
232 clk_set_parent(cpu_clk, armpll);
233 mtk_cpufreq_set_voltage(info, old_vproc);
234 return ret;
235 }
236
237 /* Set parent of CPU clock back to the original PLL. */
238 ret = clk_set_parent(cpu_clk, armpll);
239 if (ret) {
240 pr_err("cpu%d: failed to re-parent cpu clock!\n",
241 policy->cpu);
242 mtk_cpufreq_set_voltage(info, inter_vproc);
243 WARN_ON(1);
244 return ret;
245 }
246
247 /*
248 * If the new voltage is lower than the intermediate voltage or the
249 * original voltage, scale down to the new voltage.
250 */
251 if (vproc < inter_vproc || vproc < old_vproc) {
252 ret = mtk_cpufreq_set_voltage(info, vproc);
253 if (ret) {
254 pr_err("cpu%d: failed to scale down voltage!\n",
255 policy->cpu);
256 clk_set_parent(cpu_clk, info->inter_clk);
257 clk_set_rate(armpll, old_freq_hz);
258 clk_set_parent(cpu_clk, armpll);
259 return ret;
260 }
261 }
262
263 return 0;
264}
265
266static void mtk_cpufreq_ready(struct cpufreq_policy *policy)
267{
268 struct mtk_cpu_dvfs_info *info = policy->driver_data;
269 struct device_node *np = of_node_get(info->cpu_dev->of_node);
270
271 if (WARN_ON(!np))
272 return;
273
274 if (of_find_property(np, "#cooling-cells", NULL)) {
275 info->cdev = of_cpufreq_cooling_register(np,
276 policy->related_cpus);
277
278 if (IS_ERR(info->cdev)) {
279 dev_err(info->cpu_dev,
280 "running cpufreq without cooling device: %ld\n",
281 PTR_ERR(info->cdev));
282
283 info->cdev = NULL;
284 }
285 }
286
287 of_node_put(np);
288}
289
290static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu)
291{
292 struct device *cpu_dev;
293 struct regulator *proc_reg = ERR_PTR(-ENODEV);
294 struct regulator *sram_reg = ERR_PTR(-ENODEV);
295 struct clk *cpu_clk = ERR_PTR(-ENODEV);
296 struct clk *inter_clk = ERR_PTR(-ENODEV);
297 struct dev_pm_opp *opp;
298 unsigned long rate;
299 int ret;
300
301 cpu_dev = get_cpu_device(cpu);
302 if (!cpu_dev) {
303 pr_err("failed to get cpu%d device\n", cpu);
304 return -ENODEV;
305 }
306
307 cpu_clk = clk_get(cpu_dev, "cpu");
308 if (IS_ERR(cpu_clk)) {
309 if (PTR_ERR(cpu_clk) == -EPROBE_DEFER)
310 pr_warn("cpu clk for cpu%d not ready, retry.\n", cpu);
311 else
312 pr_err("failed to get cpu clk for cpu%d\n", cpu);
313
314 ret = PTR_ERR(cpu_clk);
315 return ret;
316 }
317
318 inter_clk = clk_get(cpu_dev, "intermediate");
319 if (IS_ERR(inter_clk)) {
320 if (PTR_ERR(inter_clk) == -EPROBE_DEFER)
321 pr_warn("intermediate clk for cpu%d not ready, retry.\n",
322 cpu);
323 else
324 pr_err("failed to get intermediate clk for cpu%d\n",
325 cpu);
326
327 ret = PTR_ERR(inter_clk);
328 goto out_free_resources;
329 }
330
331 proc_reg = regulator_get_exclusive(cpu_dev, "proc");
332 if (IS_ERR(proc_reg)) {
333 if (PTR_ERR(proc_reg) == -EPROBE_DEFER)
334 pr_warn("proc regulator for cpu%d not ready, retry.\n",
335 cpu);
336 else
337 pr_err("failed to get proc regulator for cpu%d\n",
338 cpu);
339
340 ret = PTR_ERR(proc_reg);
341 goto out_free_resources;
342 }
343
344 /* Both presence and absence of sram regulator are valid cases. */
345 sram_reg = regulator_get_exclusive(cpu_dev, "sram");
346
347 ret = of_init_opp_table(cpu_dev);
348 if (ret) {
349 pr_warn("no OPP table for cpu%d\n", cpu);
350 goto out_free_resources;
351 }
352
353 /* Search a safe voltage for intermediate frequency. */
354 rate = clk_get_rate(inter_clk);
355 rcu_read_lock();
356 opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate);
357 if (IS_ERR(opp)) {
358 rcu_read_unlock();
359 pr_err("failed to get intermediate opp for cpu%d\n", cpu);
360 ret = PTR_ERR(opp);
361 goto out_free_opp_table;
362 }
363 info->intermediate_voltage = dev_pm_opp_get_voltage(opp);
364 rcu_read_unlock();
365
366 info->cpu_dev = cpu_dev;
367 info->proc_reg = proc_reg;
368 info->sram_reg = IS_ERR(sram_reg) ? NULL : sram_reg;
369 info->cpu_clk = cpu_clk;
370 info->inter_clk = inter_clk;
371
372 /*
373 * If SRAM regulator is present, software "voltage tracking" is needed
374 * for this CPU power domain.
375 */
376 info->need_voltage_tracking = !IS_ERR(sram_reg);
377
378 return 0;
379
380out_free_opp_table:
381 of_free_opp_table(cpu_dev);
382
383out_free_resources:
384 if (!IS_ERR(proc_reg))
385 regulator_put(proc_reg);
386 if (!IS_ERR(sram_reg))
387 regulator_put(sram_reg);
388 if (!IS_ERR(cpu_clk))
389 clk_put(cpu_clk);
390 if (!IS_ERR(inter_clk))
391 clk_put(inter_clk);
392
393 return ret;
394}
395
396static void mtk_cpu_dvfs_info_release(struct mtk_cpu_dvfs_info *info)
397{
398 if (!IS_ERR(info->proc_reg))
399 regulator_put(info->proc_reg);
400 if (!IS_ERR(info->sram_reg))
401 regulator_put(info->sram_reg);
402 if (!IS_ERR(info->cpu_clk))
403 clk_put(info->cpu_clk);
404 if (!IS_ERR(info->inter_clk))
405 clk_put(info->inter_clk);
406
407 of_free_opp_table(info->cpu_dev);
408}
409
410static int mtk_cpufreq_init(struct cpufreq_policy *policy)
411{
412 struct mtk_cpu_dvfs_info *info;
413 struct cpufreq_frequency_table *freq_table;
414 int ret;
415
416 info = kzalloc(sizeof(*info), GFP_KERNEL);
417 if (!info)
418 return -ENOMEM;
419
420 ret = mtk_cpu_dvfs_info_init(info, policy->cpu);
421 if (ret) {
422 pr_err("%s failed to initialize dvfs info for cpu%d\n",
423 __func__, policy->cpu);
424 goto out_free_dvfs_info;
425 }
426
427 ret = dev_pm_opp_init_cpufreq_table(info->cpu_dev, &freq_table);
428 if (ret) {
429 pr_err("failed to init cpufreq table for cpu%d: %d\n",
430 policy->cpu, ret);
431 goto out_release_dvfs_info;
432 }
433
434 ret = cpufreq_table_validate_and_show(policy, freq_table);
435 if (ret) {
436 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
437 goto out_free_cpufreq_table;
438 }
439
440 /* CPUs in the same cluster share a clock and power domain. */
441 cpumask_copy(policy->cpus, &cpu_topology[policy->cpu].core_sibling);
442 policy->driver_data = info;
443 policy->clk = info->cpu_clk;
444
445 return 0;
446
447out_free_cpufreq_table:
448 dev_pm_opp_free_cpufreq_table(info->cpu_dev, &freq_table);
449
450out_release_dvfs_info:
451 mtk_cpu_dvfs_info_release(info);
452
453out_free_dvfs_info:
454 kfree(info);
455
456 return ret;
457}
458
459static int mtk_cpufreq_exit(struct cpufreq_policy *policy)
460{
461 struct mtk_cpu_dvfs_info *info = policy->driver_data;
462
463 cpufreq_cooling_unregister(info->cdev);
464 dev_pm_opp_free_cpufreq_table(info->cpu_dev, &policy->freq_table);
465 mtk_cpu_dvfs_info_release(info);
466 kfree(info);
467
468 return 0;
469}
470
471static struct cpufreq_driver mt8173_cpufreq_driver = {
472 .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK,
473 .verify = cpufreq_generic_frequency_table_verify,
474 .target_index = mtk_cpufreq_set_target,
475 .get = cpufreq_generic_get,
476 .init = mtk_cpufreq_init,
477 .exit = mtk_cpufreq_exit,
478 .ready = mtk_cpufreq_ready,
479 .name = "mtk-cpufreq",
480 .attr = cpufreq_generic_attr,
481};
482
483static int mt8173_cpufreq_probe(struct platform_device *pdev)
484{
485 int ret;
486
487 ret = cpufreq_register_driver(&mt8173_cpufreq_driver);
488 if (ret)
489 pr_err("failed to register mtk cpufreq driver\n");
490
491 return ret;
492}
493
494static struct platform_driver mt8173_cpufreq_platdrv = {
495 .driver = {
496 .name = "mt8173-cpufreq",
497 },
498 .probe = mt8173_cpufreq_probe,
499};
500
501static int mt8173_cpufreq_driver_init(void)
502{
503 struct platform_device *pdev;
504 int err;
505
506 if (!of_machine_is_compatible("mediatek,mt8173"))
507 return -ENODEV;
508
509 err = platform_driver_register(&mt8173_cpufreq_platdrv);
510 if (err)
511 return err;
512
513 /*
514 * Since there's no place to hold device registration code and no
515 * device tree based way to match cpufreq driver yet, both the driver
516 * and the device registration codes are put here to handle defer
517 * probing.
518 */
519 pdev = platform_device_register_simple("mt8173-cpufreq", -1, NULL, 0);
520 if (IS_ERR(pdev)) {
521 pr_err("failed to register mtk-cpufreq platform device\n");
522 return PTR_ERR(pdev);
523 }
524
525 return 0;
526}
527device_initcall(mt8173_cpufreq_driver_init);
diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c
index 37c5742482d8..c1ae1999770a 100644
--- a/drivers/cpufreq/powernow-k7.c
+++ b/drivers/cpufreq/powernow-k7.c
@@ -421,7 +421,7 @@ static int powernow_acpi_init(void)
421 return 0; 421 return 0;
422 422
423err2: 423err2:
424 acpi_processor_unregister_performance(acpi_processor_perf, 0); 424 acpi_processor_unregister_performance(0);
425err1: 425err1:
426 free_cpumask_var(acpi_processor_perf->shared_cpu_map); 426 free_cpumask_var(acpi_processor_perf->shared_cpu_map);
427err05: 427err05:
@@ -661,7 +661,7 @@ static int powernow_cpu_exit(struct cpufreq_policy *policy)
661{ 661{
662#ifdef CONFIG_X86_POWERNOW_K7_ACPI 662#ifdef CONFIG_X86_POWERNOW_K7_ACPI
663 if (acpi_processor_perf) { 663 if (acpi_processor_perf) {
664 acpi_processor_unregister_performance(acpi_processor_perf, 0); 664 acpi_processor_unregister_performance(0);
665 free_cpumask_var(acpi_processor_perf->shared_cpu_map); 665 free_cpumask_var(acpi_processor_perf->shared_cpu_map);
666 kfree(acpi_processor_perf); 666 kfree(acpi_processor_perf);
667 } 667 }
diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c
index 5c035d04d827..0b5bf135b090 100644
--- a/drivers/cpufreq/powernow-k8.c
+++ b/drivers/cpufreq/powernow-k8.c
@@ -795,7 +795,7 @@ err_out_mem:
795 kfree(powernow_table); 795 kfree(powernow_table);
796 796
797err_out: 797err_out:
798 acpi_processor_unregister_performance(&data->acpi_data, data->cpu); 798 acpi_processor_unregister_performance(data->cpu);
799 799
800 /* data->acpi_data.state_count informs us at ->exit() 800 /* data->acpi_data.state_count informs us at ->exit()
801 * whether ACPI was used */ 801 * whether ACPI was used */
@@ -863,8 +863,7 @@ static int fill_powernow_table_fidvid(struct powernow_k8_data *data,
863static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data) 863static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data)
864{ 864{
865 if (data->acpi_data.state_count) 865 if (data->acpi_data.state_count)
866 acpi_processor_unregister_performance(&data->acpi_data, 866 acpi_processor_unregister_performance(data->cpu);
867 data->cpu);
868 free_cpumask_var(data->acpi_data.shared_cpu_map); 867 free_cpumask_var(data->acpi_data.shared_cpu_map);
869} 868}
870 869
diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c
index ebef0d8279c7..64994e10638e 100644
--- a/drivers/cpufreq/powernv-cpufreq.c
+++ b/drivers/cpufreq/powernv-cpufreq.c
@@ -27,20 +27,31 @@
27#include <linux/smp.h> 27#include <linux/smp.h>
28#include <linux/of.h> 28#include <linux/of.h>
29#include <linux/reboot.h> 29#include <linux/reboot.h>
30#include <linux/slab.h>
30 31
31#include <asm/cputhreads.h> 32#include <asm/cputhreads.h>
32#include <asm/firmware.h> 33#include <asm/firmware.h>
33#include <asm/reg.h> 34#include <asm/reg.h>
34#include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */ 35#include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */
36#include <asm/opal.h>
35 37
36#define POWERNV_MAX_PSTATES 256 38#define POWERNV_MAX_PSTATES 256
37#define PMSR_PSAFE_ENABLE (1UL << 30) 39#define PMSR_PSAFE_ENABLE (1UL << 30)
38#define PMSR_SPR_EM_DISABLE (1UL << 31) 40#define PMSR_SPR_EM_DISABLE (1UL << 31)
39#define PMSR_MAX(x) ((x >> 32) & 0xFF) 41#define PMSR_MAX(x) ((x >> 32) & 0xFF)
40#define PMSR_LP(x) ((x >> 48) & 0xFF)
41 42
42static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1]; 43static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1];
43static bool rebooting, throttled; 44static bool rebooting, throttled, occ_reset;
45
46static struct chip {
47 unsigned int id;
48 bool throttled;
49 cpumask_t mask;
50 struct work_struct throttle;
51 bool restore;
52} *chips;
53
54static int nr_chips;
44 55
45/* 56/*
46 * Note: The set of pstates consists of contiguous integers, the 57 * Note: The set of pstates consists of contiguous integers, the
@@ -298,28 +309,35 @@ static inline unsigned int get_nominal_index(void)
298 return powernv_pstate_info.max - powernv_pstate_info.nominal; 309 return powernv_pstate_info.max - powernv_pstate_info.nominal;
299} 310}
300 311
301static void powernv_cpufreq_throttle_check(unsigned int cpu) 312static void powernv_cpufreq_throttle_check(void *data)
302{ 313{
314 unsigned int cpu = smp_processor_id();
303 unsigned long pmsr; 315 unsigned long pmsr;
304 int pmsr_pmax, pmsr_lp; 316 int pmsr_pmax, i;
305 317
306 pmsr = get_pmspr(SPRN_PMSR); 318 pmsr = get_pmspr(SPRN_PMSR);
307 319
320 for (i = 0; i < nr_chips; i++)
321 if (chips[i].id == cpu_to_chip_id(cpu))
322 break;
323
308 /* Check for Pmax Capping */ 324 /* Check for Pmax Capping */
309 pmsr_pmax = (s8)PMSR_MAX(pmsr); 325 pmsr_pmax = (s8)PMSR_MAX(pmsr);
310 if (pmsr_pmax != powernv_pstate_info.max) { 326 if (pmsr_pmax != powernv_pstate_info.max) {
311 throttled = true; 327 if (chips[i].throttled)
312 pr_info("CPU %d Pmax is reduced to %d\n", cpu, pmsr_pmax); 328 goto next;
313 pr_info("Max allowed Pstate is capped\n"); 329 chips[i].throttled = true;
330 pr_info("CPU %d on Chip %u has Pmax reduced to %d\n", cpu,
331 chips[i].id, pmsr_pmax);
332 } else if (chips[i].throttled) {
333 chips[i].throttled = false;
334 pr_info("CPU %d on Chip %u has Pmax restored to %d\n", cpu,
335 chips[i].id, pmsr_pmax);
314 } 336 }
315 337
316 /* 338 /* Check if Psafe_mode_active is set in PMSR. */
317 * Check for Psafe by reading LocalPstate 339next:
318 * or check if Psafe_mode_active is set in PMSR. 340 if (pmsr & PMSR_PSAFE_ENABLE) {
319 */
320 pmsr_lp = (s8)PMSR_LP(pmsr);
321 if ((pmsr_lp < powernv_pstate_info.min) ||
322 (pmsr & PMSR_PSAFE_ENABLE)) {
323 throttled = true; 341 throttled = true;
324 pr_info("Pstate set to safe frequency\n"); 342 pr_info("Pstate set to safe frequency\n");
325 } 343 }
@@ -350,7 +368,7 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
350 return 0; 368 return 0;
351 369
352 if (!throttled) 370 if (!throttled)
353 powernv_cpufreq_throttle_check(smp_processor_id()); 371 powernv_cpufreq_throttle_check(NULL);
354 372
355 freq_data.pstate_id = powernv_freqs[new_index].driver_data; 373 freq_data.pstate_id = powernv_freqs[new_index].driver_data;
356 374
@@ -395,6 +413,119 @@ static struct notifier_block powernv_cpufreq_reboot_nb = {
395 .notifier_call = powernv_cpufreq_reboot_notifier, 413 .notifier_call = powernv_cpufreq_reboot_notifier,
396}; 414};
397 415
416void powernv_cpufreq_work_fn(struct work_struct *work)
417{
418 struct chip *chip = container_of(work, struct chip, throttle);
419 unsigned int cpu;
420 cpumask_var_t mask;
421
422 smp_call_function_any(&chip->mask,
423 powernv_cpufreq_throttle_check, NULL, 0);
424
425 if (!chip->restore)
426 return;
427
428 chip->restore = false;
429 cpumask_copy(mask, &chip->mask);
430 for_each_cpu_and(cpu, mask, cpu_online_mask) {
431 int index, tcpu;
432 struct cpufreq_policy policy;
433
434 cpufreq_get_policy(&policy, cpu);
435 cpufreq_frequency_table_target(&policy, policy.freq_table,
436 policy.cur,
437 CPUFREQ_RELATION_C, &index);
438 powernv_cpufreq_target_index(&policy, index);
439 for_each_cpu(tcpu, policy.cpus)
440 cpumask_clear_cpu(tcpu, mask);
441 }
442}
443
444static char throttle_reason[][30] = {
445 "No throttling",
446 "Power Cap",
447 "Processor Over Temperature",
448 "Power Supply Failure",
449 "Over Current",
450 "OCC Reset"
451 };
452
453static int powernv_cpufreq_occ_msg(struct notifier_block *nb,
454 unsigned long msg_type, void *_msg)
455{
456 struct opal_msg *msg = _msg;
457 struct opal_occ_msg omsg;
458 int i;
459
460 if (msg_type != OPAL_MSG_OCC)
461 return 0;
462
463 omsg.type = be64_to_cpu(msg->params[0]);
464
465 switch (omsg.type) {
466 case OCC_RESET:
467 occ_reset = true;
468 pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n");
469 /*
470 * powernv_cpufreq_throttle_check() is called in
471 * target() callback which can detect the throttle state
472 * for governors like ondemand.
473 * But static governors will not call target() often thus
474 * report throttling here.
475 */
476 if (!throttled) {
477 throttled = true;
478 pr_crit("CPU frequency is throttled for duration\n");
479 }
480
481 break;
482 case OCC_LOAD:
483 pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n");
484 break;
485 case OCC_THROTTLE:
486 omsg.chip = be64_to_cpu(msg->params[1]);
487 omsg.throttle_status = be64_to_cpu(msg->params[2]);
488
489 if (occ_reset) {
490 occ_reset = false;
491 throttled = false;
492 pr_info("OCC Active, CPU frequency is no longer throttled\n");
493
494 for (i = 0; i < nr_chips; i++) {
495 chips[i].restore = true;
496 schedule_work(&chips[i].throttle);
497 }
498
499 return 0;
500 }
501
502 if (omsg.throttle_status &&
503 omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS)
504 pr_info("OCC: Chip %u Pmax reduced due to %s\n",
505 (unsigned int)omsg.chip,
506 throttle_reason[omsg.throttle_status]);
507 else if (!omsg.throttle_status)
508 pr_info("OCC: Chip %u %s\n", (unsigned int)omsg.chip,
509 throttle_reason[omsg.throttle_status]);
510 else
511 return 0;
512
513 for (i = 0; i < nr_chips; i++)
514 if (chips[i].id == omsg.chip) {
515 if (!omsg.throttle_status)
516 chips[i].restore = true;
517 schedule_work(&chips[i].throttle);
518 }
519 }
520 return 0;
521}
522
523static struct notifier_block powernv_cpufreq_opal_nb = {
524 .notifier_call = powernv_cpufreq_occ_msg,
525 .next = NULL,
526 .priority = 0,
527};
528
398static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy) 529static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy)
399{ 530{
400 struct powernv_smp_call_data freq_data; 531 struct powernv_smp_call_data freq_data;
@@ -414,6 +545,36 @@ static struct cpufreq_driver powernv_cpufreq_driver = {
414 .attr = powernv_cpu_freq_attr, 545 .attr = powernv_cpu_freq_attr,
415}; 546};
416 547
548static int init_chip_info(void)
549{
550 unsigned int chip[256];
551 unsigned int cpu, i;
552 unsigned int prev_chip_id = UINT_MAX;
553
554 for_each_possible_cpu(cpu) {
555 unsigned int id = cpu_to_chip_id(cpu);
556
557 if (prev_chip_id != id) {
558 prev_chip_id = id;
559 chip[nr_chips++] = id;
560 }
561 }
562
563 chips = kmalloc_array(nr_chips, sizeof(struct chip), GFP_KERNEL);
564 if (!chips)
565 return -ENOMEM;
566
567 for (i = 0; i < nr_chips; i++) {
568 chips[i].id = chip[i];
569 chips[i].throttled = false;
570 cpumask_copy(&chips[i].mask, cpumask_of_node(chip[i]));
571 INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn);
572 chips[i].restore = false;
573 }
574
575 return 0;
576}
577
417static int __init powernv_cpufreq_init(void) 578static int __init powernv_cpufreq_init(void)
418{ 579{
419 int rc = 0; 580 int rc = 0;
@@ -429,7 +590,13 @@ static int __init powernv_cpufreq_init(void)
429 return rc; 590 return rc;
430 } 591 }
431 592
593 /* Populate chip info */
594 rc = init_chip_info();
595 if (rc)
596 return rc;
597
432 register_reboot_notifier(&powernv_cpufreq_reboot_nb); 598 register_reboot_notifier(&powernv_cpufreq_reboot_nb);
599 opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb);
433 return cpufreq_register_driver(&powernv_cpufreq_driver); 600 return cpufreq_register_driver(&powernv_cpufreq_driver);
434} 601}
435module_init(powernv_cpufreq_init); 602module_init(powernv_cpufreq_init);
@@ -437,6 +604,8 @@ module_init(powernv_cpufreq_init);
437static void __exit powernv_cpufreq_exit(void) 604static void __exit powernv_cpufreq_exit(void)
438{ 605{
439 unregister_reboot_notifier(&powernv_cpufreq_reboot_nb); 606 unregister_reboot_notifier(&powernv_cpufreq_reboot_nb);
607 opal_message_notifier_unregister(OPAL_MSG_OCC,
608 &powernv_cpufreq_opal_nb);
440 cpufreq_unregister_driver(&powernv_cpufreq_driver); 609 cpufreq_unregister_driver(&powernv_cpufreq_driver);
441} 610}
442module_exit(powernv_cpufreq_exit); 611module_exit(powernv_cpufreq_exit);
diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
index d29e8da396a0..7969f7690498 100644
--- a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
+++ b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
@@ -97,8 +97,8 @@ static int pmi_notifier(struct notifier_block *nb,
97 struct cpufreq_frequency_table *cbe_freqs; 97 struct cpufreq_frequency_table *cbe_freqs;
98 u8 node; 98 u8 node;
99 99
100 /* Should this really be called for CPUFREQ_ADJUST, CPUFREQ_INCOMPATIBLE 100 /* Should this really be called for CPUFREQ_ADJUST and CPUFREQ_NOTIFY
101 * and CPUFREQ_NOTIFY policy events?) 101 * policy events?)
102 */ 102 */
103 if (event == CPUFREQ_START) 103 if (event == CPUFREQ_START)
104 return 0; 104 return 0;
diff --git a/drivers/cpufreq/sfi-cpufreq.c b/drivers/cpufreq/sfi-cpufreq.c
index ffa3389e535b..992ce6f9abec 100644
--- a/drivers/cpufreq/sfi-cpufreq.c
+++ b/drivers/cpufreq/sfi-cpufreq.c
@@ -45,12 +45,10 @@ static int sfi_parse_freq(struct sfi_table_header *table)
45 pentry = (struct sfi_freq_table_entry *)sb->pentry; 45 pentry = (struct sfi_freq_table_entry *)sb->pentry;
46 totallen = num_freq_table_entries * sizeof(*pentry); 46 totallen = num_freq_table_entries * sizeof(*pentry);
47 47
48 sfi_cpufreq_array = kzalloc(totallen, GFP_KERNEL); 48 sfi_cpufreq_array = kmemdup(pentry, totallen, GFP_KERNEL);
49 if (!sfi_cpufreq_array) 49 if (!sfi_cpufreq_array)
50 return -ENOMEM; 50 return -ENOMEM;
51 51
52 memcpy(sfi_cpufreq_array, pentry, totallen);
53
54 return 0; 52 return 0;
55} 53}
56 54
diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c
index 4ab7a2156672..15d3214aaa00 100644
--- a/drivers/cpufreq/speedstep-lib.c
+++ b/drivers/cpufreq/speedstep-lib.c
@@ -386,7 +386,7 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor,
386 unsigned int prev_speed; 386 unsigned int prev_speed;
387 unsigned int ret = 0; 387 unsigned int ret = 0;
388 unsigned long flags; 388 unsigned long flags;
389 struct timeval tv1, tv2; 389 ktime_t tv1, tv2;
390 390
391 if ((!processor) || (!low_speed) || (!high_speed) || (!set_state)) 391 if ((!processor) || (!low_speed) || (!high_speed) || (!set_state))
392 return -EINVAL; 392 return -EINVAL;
@@ -415,14 +415,14 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor,
415 415
416 /* start latency measurement */ 416 /* start latency measurement */
417 if (transition_latency) 417 if (transition_latency)
418 do_gettimeofday(&tv1); 418 tv1 = ktime_get();
419 419
420 /* switch to high state */ 420 /* switch to high state */
421 set_state(SPEEDSTEP_HIGH); 421 set_state(SPEEDSTEP_HIGH);
422 422
423 /* end latency measurement */ 423 /* end latency measurement */
424 if (transition_latency) 424 if (transition_latency)
425 do_gettimeofday(&tv2); 425 tv2 = ktime_get();
426 426
427 *high_speed = speedstep_get_frequency(processor); 427 *high_speed = speedstep_get_frequency(processor);
428 if (!*high_speed) { 428 if (!*high_speed) {
@@ -442,8 +442,7 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor,
442 set_state(SPEEDSTEP_LOW); 442 set_state(SPEEDSTEP_LOW);
443 443
444 if (transition_latency) { 444 if (transition_latency) {
445 *transition_latency = (tv2.tv_sec - tv1.tv_sec) * USEC_PER_SEC + 445 *transition_latency = ktime_to_us(ktime_sub(tv2, tv1));
446 tv2.tv_usec - tv1.tv_usec;
447 pr_debug("transition latency is %u uSec\n", *transition_latency); 446 pr_debug("transition latency is %u uSec\n", *transition_latency);
448 447
449 /* convert uSec to nSec and add 20% for safety reasons */ 448 /* convert uSec to nSec and add 20% for safety reasons */
diff --git a/drivers/video/fbdev/pxafb.c b/drivers/video/fbdev/pxafb.c
index 7245611ec963..94813af97f09 100644
--- a/drivers/video/fbdev/pxafb.c
+++ b/drivers/video/fbdev/pxafb.c
@@ -1668,7 +1668,6 @@ pxafb_freq_policy(struct notifier_block *nb, unsigned long val, void *data)
1668 1668
1669 switch (val) { 1669 switch (val) {
1670 case CPUFREQ_ADJUST: 1670 case CPUFREQ_ADJUST:
1671 case CPUFREQ_INCOMPATIBLE:
1672 pr_debug("min dma period: %d ps, " 1671 pr_debug("min dma period: %d ps, "
1673 "new clock %d kHz\n", pxafb_display_dma_period(var), 1672 "new clock %d kHz\n", pxafb_display_dma_period(var),
1674 policy->max); 1673 policy->max);
diff --git a/drivers/video/fbdev/sa1100fb.c b/drivers/video/fbdev/sa1100fb.c
index 89dd7e02197f..dcf774c15889 100644
--- a/drivers/video/fbdev/sa1100fb.c
+++ b/drivers/video/fbdev/sa1100fb.c
@@ -1042,7 +1042,6 @@ sa1100fb_freq_policy(struct notifier_block *nb, unsigned long val,
1042 1042
1043 switch (val) { 1043 switch (val) {
1044 case CPUFREQ_ADJUST: 1044 case CPUFREQ_ADJUST:
1045 case CPUFREQ_INCOMPATIBLE:
1046 dev_dbg(fbi->dev, "min dma period: %d ps, " 1045 dev_dbg(fbi->dev, "min dma period: %d ps, "
1047 "new clock %d kHz\n", sa1100fb_min_dma_period(fbi), 1046 "new clock %d kHz\n", sa1100fb_min_dma_period(fbi),
1048 policy->max); 1047 policy->max);
diff --git a/drivers/xen/xen-acpi-processor.c b/drivers/xen/xen-acpi-processor.c
index 59fc190f1e92..70fa438000af 100644
--- a/drivers/xen/xen-acpi-processor.c
+++ b/drivers/xen/xen-acpi-processor.c
@@ -560,11 +560,9 @@ static int __init xen_acpi_processor_init(void)
560 560
561 return 0; 561 return 0;
562err_unregister: 562err_unregister:
563 for_each_possible_cpu(i) { 563 for_each_possible_cpu(i)
564 struct acpi_processor_performance *perf; 564 acpi_processor_unregister_performance(i);
565 perf = per_cpu_ptr(acpi_perf_data, i); 565
566 acpi_processor_unregister_performance(perf, i);
567 }
568err_out: 566err_out:
569 /* Freeing a NULL pointer is OK: alloc_percpu zeroes. */ 567 /* Freeing a NULL pointer is OK: alloc_percpu zeroes. */
570 free_acpi_perf_data(); 568 free_acpi_perf_data();
@@ -579,11 +577,9 @@ static void __exit xen_acpi_processor_exit(void)
579 kfree(acpi_ids_done); 577 kfree(acpi_ids_done);
580 kfree(acpi_id_present); 578 kfree(acpi_id_present);
581 kfree(acpi_id_cst_present); 579 kfree(acpi_id_cst_present);
582 for_each_possible_cpu(i) { 580 for_each_possible_cpu(i)
583 struct acpi_processor_performance *perf; 581 acpi_processor_unregister_performance(i);
584 perf = per_cpu_ptr(acpi_perf_data, i); 582
585 acpi_processor_unregister_performance(perf, i);
586 }
587 free_acpi_perf_data(); 583 free_acpi_perf_data();
588} 584}
589 585
diff --git a/include/acpi/processor.h b/include/acpi/processor.h
index 2c4e7a9c1725..ff5f135f16b1 100644
--- a/include/acpi/processor.h
+++ b/include/acpi/processor.h
@@ -228,10 +228,7 @@ extern int acpi_processor_preregister_performance(struct
228 228
229extern int acpi_processor_register_performance(struct acpi_processor_performance 229extern int acpi_processor_register_performance(struct acpi_processor_performance
230 *performance, unsigned int cpu); 230 *performance, unsigned int cpu);
231extern void acpi_processor_unregister_performance(struct 231extern void acpi_processor_unregister_performance(unsigned int cpu);
232 acpi_processor_performance
233 *performance,
234 unsigned int cpu);
235 232
236/* note: this locks both the calling module and the processor module 233/* note: this locks both the calling module and the processor module
237 if a _PPC object exists, rmmod is disallowed then */ 234 if a _PPC object exists, rmmod is disallowed then */
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index bde1e567b3a9..6ff6a4d95eea 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -51,11 +51,9 @@ struct cpufreq_cpuinfo {
51 unsigned int transition_latency; 51 unsigned int transition_latency;
52}; 52};
53 53
54struct cpufreq_real_policy { 54struct cpufreq_user_policy {
55 unsigned int min; /* in kHz */ 55 unsigned int min; /* in kHz */
56 unsigned int max; /* in kHz */ 56 unsigned int max; /* in kHz */
57 unsigned int policy; /* see above */
58 struct cpufreq_governor *governor; /* see below */
59}; 57};
60 58
61struct cpufreq_policy { 59struct cpufreq_policy {
@@ -88,7 +86,7 @@ struct cpufreq_policy {
88 struct work_struct update; /* if update_policy() needs to be 86 struct work_struct update; /* if update_policy() needs to be
89 * called, but you're in IRQ context */ 87 * called, but you're in IRQ context */
90 88
91 struct cpufreq_real_policy user_policy; 89 struct cpufreq_user_policy user_policy;
92 struct cpufreq_frequency_table *freq_table; 90 struct cpufreq_frequency_table *freq_table;
93 91
94 struct list_head policy_list; 92 struct list_head policy_list;
@@ -369,11 +367,10 @@ static inline void cpufreq_resume(void) {}
369 367
370/* Policy Notifiers */ 368/* Policy Notifiers */
371#define CPUFREQ_ADJUST (0) 369#define CPUFREQ_ADJUST (0)
372#define CPUFREQ_INCOMPATIBLE (1) 370#define CPUFREQ_NOTIFY (1)
373#define CPUFREQ_NOTIFY (2) 371#define CPUFREQ_START (2)
374#define CPUFREQ_START (3) 372#define CPUFREQ_CREATE_POLICY (3)
375#define CPUFREQ_CREATE_POLICY (4) 373#define CPUFREQ_REMOVE_POLICY (4)
376#define CPUFREQ_REMOVE_POLICY (5)
377 374
378#ifdef CONFIG_CPU_FREQ 375#ifdef CONFIG_CPU_FREQ
379int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list); 376int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list);
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-04 21:32:47 -0500