aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2009-09-11 16:22:30 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2009-09-11 16:22:30 -0400
commitb9356c53ba2f593081e5aa45eb67adcce243d1c0 (patch)
treedebf2b72d5241dbaa5b4f2088feb0a2584388792
parentd90a7e86401ffea2163a4337f3a47f3909c4e255 (diff)
parent4680e64a88c4ce2c4e736dade99233e3def13fa7 (diff)
Merge branch 'oprofile-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'oprofile-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (55 commits) arch/x86/oprofile/op_model_amd.c: fix op_amd_handle_ibs() return type Revert "x86: oprofile/op_model_amd.c set return values for op_amd_handle_ibs()" x86/oprofile: Small coding style fixes x86/oprofile: Add counter reservation check for virtual counters x86/oprofile: Implement op_x86_virt_to_phys() oprofile: Adding switch counter to oprofile statistic variables x86/oprofile: Implement mux_clone() x86/oprofile: Enable multiplexing only if the model supports it x86/oprofile: Add function has_mux() to check multiplexing support x86/oprofile: Modify initialization of num_virt_counters x86/oprofile: Remove unused num_virt_controls from struct op_x86_model_spec x86/oprofile: Remove const qualifier from struct op_x86_model_spec x86/oprofile: Moving nmi_cpu_switch() in nmi_int.c x86/oprofile: Moving nmi_cpu_save/restore_mpx_registers() in nmi_int.c x86/oprofile: Moving nmi_setup_cpu_mux() in nmi_int.c x86/oprofile: Implement multiplexing setup/shutdown functions oprofile: Grouping multiplexing code in op_model_amd.c oprofile: Introduce op_x86_phys_to_virt() oprofile: Grouping multiplexing code in oprof.c oprofile: Remove oprofile_multiplexing_init() ...
Diffstat
-rw-r--r--arch/Kconfig12
-rw-r--r--arch/x86/oprofile/nmi_int.c404
-rw-r--r--arch/x86/oprofile/op_counter.h2
-rw-r--r--arch/x86/oprofile/op_model_amd.c372
-rw-r--r--arch/x86/oprofile/op_model_p4.c72
-rw-r--r--arch/x86/oprofile/op_model_ppro.c101
-rw-r--r--arch/x86/oprofile/op_x86_model.h59
-rw-r--r--drivers/oprofile/cpu_buffer.c16
-rw-r--r--drivers/oprofile/oprof.c71
-rw-r--r--drivers/oprofile/oprof.h3
-rw-r--r--drivers/oprofile/oprofile_files.c46
-rw-r--r--drivers/oprofile/oprofile_stats.c5
-rw-r--r--drivers/oprofile/oprofile_stats.h1
-rw-r--r--include/linux/oprofile.h5
14 files changed, 766 insertions, 403 deletions
diff --git a/arch/Kconfig b/arch/Kconfig
index 99193b160232..beea3ccebb5e 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -30,6 +30,18 @@ config OPROFILE_IBS
30 30
31 If unsure, say N. 31 If unsure, say N.
32 32
33config OPROFILE_EVENT_MULTIPLEX
34 bool "OProfile multiplexing support (EXPERIMENTAL)"
35 default n
36 depends on OPROFILE && X86
37 help
38 The number of hardware counters is limited. The multiplexing
39 feature enables OProfile to gather more events than counters
40 are provided by the hardware. This is realized by switching
41 between events at an user specified time interval.
42
43 If unsure, say N.
44
33config HAVE_OPROFILE 45config HAVE_OPROFILE
34 bool 46 bool
35 47
diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index 89b9a5cd63da..cb88b1a0bd5f 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -1,11 +1,14 @@
1/** 1/**
2 * @file nmi_int.c 2 * @file nmi_int.c
3 * 3 *
4 * @remark Copyright 2002-2008 OProfile authors 4 * @remark Copyright 2002-2009 OProfile authors
5 * @remark Read the file COPYING 5 * @remark Read the file COPYING
6 * 6 *
7 * @author John Levon <levon@movementarian.org> 7 * @author John Levon <levon@movementarian.org>
8 * @author Robert Richter <robert.richter@amd.com> 8 * @author Robert Richter <robert.richter@amd.com>
9 * @author Barry Kasindorf <barry.kasindorf@amd.com>
10 * @author Jason Yeh <jason.yeh@amd.com>
11 * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
9 */ 12 */
10 13
11#include <linux/init.h> 14#include <linux/init.h>
@@ -24,13 +27,35 @@
24#include "op_counter.h" 27#include "op_counter.h"
25#include "op_x86_model.h" 28#include "op_x86_model.h"
26 29
27static struct op_x86_model_spec const *model; 30static struct op_x86_model_spec *model;
28static DEFINE_PER_CPU(struct op_msrs, cpu_msrs); 31static DEFINE_PER_CPU(struct op_msrs, cpu_msrs);
29static DEFINE_PER_CPU(unsigned long, saved_lvtpc); 32static DEFINE_PER_CPU(unsigned long, saved_lvtpc);
30 33
31/* 0 == registered but off, 1 == registered and on */ 34/* 0 == registered but off, 1 == registered and on */
32static int nmi_enabled = 0; 35static int nmi_enabled = 0;
33 36
37struct op_counter_config counter_config[OP_MAX_COUNTER];
38
39/* common functions */
40
41u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
42 struct op_counter_config *counter_config)
43{
44 u64 val = 0;
45 u16 event = (u16)counter_config->event;
46
47 val |= ARCH_PERFMON_EVENTSEL_INT;
48 val |= counter_config->user ? ARCH_PERFMON_EVENTSEL_USR : 0;
49 val |= counter_config->kernel ? ARCH_PERFMON_EVENTSEL_OS : 0;
50 val |= (counter_config->unit_mask & 0xFF) << 8;
51 event &= model->event_mask ? model->event_mask : 0xFF;
52 val |= event & 0xFF;
53 val |= (event & 0x0F00) << 24;
54
55 return val;
56}
57
58
34static int profile_exceptions_notify(struct notifier_block *self, 59static int profile_exceptions_notify(struct notifier_block *self,
35 unsigned long val, void *data) 60 unsigned long val, void *data)
36{ 61{
@@ -52,36 +77,214 @@ static int profile_exceptions_notify(struct notifier_block *self,
52 77
53static void nmi_cpu_save_registers(struct op_msrs *msrs) 78static void nmi_cpu_save_registers(struct op_msrs *msrs)
54{ 79{
55 unsigned int const nr_ctrs = model->num_counters;
56 unsigned int const nr_ctrls = model->num_controls;
57 struct op_msr *counters = msrs->counters; 80 struct op_msr *counters = msrs->counters;
58 struct op_msr *controls = msrs->controls; 81 struct op_msr *controls = msrs->controls;
59 unsigned int i; 82 unsigned int i;
60 83
61 for (i = 0; i < nr_ctrs; ++i) { 84 for (i = 0; i < model->num_counters; ++i) {
62 if (counters[i].addr) { 85 if (counters[i].addr)
63 rdmsr(counters[i].addr, 86 rdmsrl(counters[i].addr, counters[i].saved);
64 counters[i].saved.low, 87 }
65 counters[i].saved.high); 88
66 } 89 for (i = 0; i < model->num_controls; ++i) {
90 if (controls[i].addr)
91 rdmsrl(controls[i].addr, controls[i].saved);
92 }
93}
94
95static void nmi_cpu_start(void *dummy)
96{
97 struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
98 model->start(msrs);
99}
100
101static int nmi_start(void)
102{
103 on_each_cpu(nmi_cpu_start, NULL, 1);
104 return 0;
105}
106
107static void nmi_cpu_stop(void *dummy)
108{
109 struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
110 model->stop(msrs);
111}
112
113static void nmi_stop(void)
114{
115 on_each_cpu(nmi_cpu_stop, NULL, 1);
116}
117
118#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
119
120static DEFINE_PER_CPU(int, switch_index);
121
122static inline int has_mux(void)
123{
124 return !!model->switch_ctrl;
125}
126
127inline int op_x86_phys_to_virt(int phys)
128{
129 return __get_cpu_var(switch_index) + phys;
130}
131
132inline int op_x86_virt_to_phys(int virt)
133{
134 return virt % model->num_counters;
135}
136
137static void nmi_shutdown_mux(void)
138{
139 int i;
140
141 if (!has_mux())
142 return;
143
144 for_each_possible_cpu(i) {
145 kfree(per_cpu(cpu_msrs, i).multiplex);
146 per_cpu(cpu_msrs, i).multiplex = NULL;
147 per_cpu(switch_index, i) = 0;
67 } 148 }
149}
150
151static int nmi_setup_mux(void)
152{
153 size_t multiplex_size =
154 sizeof(struct op_msr) * model->num_virt_counters;
155 int i;
156
157 if (!has_mux())
158 return 1;
159
160 for_each_possible_cpu(i) {
161 per_cpu(cpu_msrs, i).multiplex =
162 kmalloc(multiplex_size, GFP_KERNEL);
163 if (!per_cpu(cpu_msrs, i).multiplex)
164 return 0;
165 }
166
167 return 1;
168}
169
170static void nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs)
171{
172 int i;
173 struct op_msr *multiplex = msrs->multiplex;
174
175 if (!has_mux())
176 return;
68 177
69 for (i = 0; i < nr_ctrls; ++i) { 178 for (i = 0; i < model->num_virt_counters; ++i) {
70 if (controls[i].addr) { 179 if (counter_config[i].enabled) {
71 rdmsr(controls[i].addr, 180 multiplex[i].saved = -(u64)counter_config[i].count;
72 controls[i].saved.low, 181 } else {
73 controls[i].saved.high); 182 multiplex[i].addr = 0;
183 multiplex[i].saved = 0;
74 } 184 }
75 } 185 }
186
187 per_cpu(switch_index, cpu) = 0;
188}
189
190static void nmi_cpu_save_mpx_registers(struct op_msrs *msrs)
191{
192 struct op_msr *multiplex = msrs->multiplex;
193 int i;
194
195 for (i = 0; i < model->num_counters; ++i) {
196 int virt = op_x86_phys_to_virt(i);
197 if (multiplex[virt].addr)
198 rdmsrl(multiplex[virt].addr, multiplex[virt].saved);
199 }
200}
201
202static void nmi_cpu_restore_mpx_registers(struct op_msrs *msrs)
203{
204 struct op_msr *multiplex = msrs->multiplex;
205 int i;
206
207 for (i = 0; i < model->num_counters; ++i) {
208 int virt = op_x86_phys_to_virt(i);
209 if (multiplex[virt].addr)
210 wrmsrl(multiplex[virt].addr, multiplex[virt].saved);
211 }
76} 212}
77 213
78static void nmi_save_registers(void *dummy) 214static void nmi_cpu_switch(void *dummy)
79{ 215{
80 int cpu = smp_processor_id(); 216 int cpu = smp_processor_id();
217 int si = per_cpu(switch_index, cpu);
81 struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu); 218 struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
82 nmi_cpu_save_registers(msrs); 219
220 nmi_cpu_stop(NULL);
221 nmi_cpu_save_mpx_registers(msrs);
222
223 /* move to next set */
224 si += model->num_counters;
225 if ((si > model->num_virt_counters) || (counter_config[si].count == 0))
226 per_cpu(switch_index, cpu) = 0;
227 else
228 per_cpu(switch_index, cpu) = si;
229
230 model->switch_ctrl(model, msrs);
231 nmi_cpu_restore_mpx_registers(msrs);
232
233 nmi_cpu_start(NULL);
234}
235
236
237/*
238 * Quick check to see if multiplexing is necessary.
239 * The check should be sufficient since counters are used
240 * in ordre.
241 */
242static int nmi_multiplex_on(void)
243{
244 return counter_config[model->num_counters].count ? 0 : -EINVAL;
245}
246
247static int nmi_switch_event(void)
248{
249 if (!has_mux())
250 return -ENOSYS; /* not implemented */
251 if (nmi_multiplex_on() < 0)
252 return -EINVAL; /* not necessary */
253
254 on_each_cpu(nmi_cpu_switch, NULL, 1);
255
256 return 0;
257}
258
259static inline void mux_init(struct oprofile_operations *ops)
260{
261 if (has_mux())
262 ops->switch_events = nmi_switch_event;
263}
264
265static void mux_clone(int cpu)
266{
267 if (!has_mux())
268 return;
269
270 memcpy(per_cpu(cpu_msrs, cpu).multiplex,
271 per_cpu(cpu_msrs, 0).multiplex,
272 sizeof(struct op_msr) * model->num_virt_counters);
83} 273}
84 274
275#else
276
277inline int op_x86_phys_to_virt(int phys) { return phys; }
278inline int op_x86_virt_to_phys(int virt) { return virt; }
279static inline void nmi_shutdown_mux(void) { }
280static inline int nmi_setup_mux(void) { return 1; }
281static inline void
282nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs) { }
283static inline void mux_init(struct oprofile_operations *ops) { }
284static void mux_clone(int cpu) { }
285
286#endif
287
85static void free_msrs(void) 288static void free_msrs(void)
86{ 289{
87 int i; 290 int i;
@@ -95,38 +298,32 @@ static void free_msrs(void)
95 298
96static int allocate_msrs(void) 299static int allocate_msrs(void)
97{ 300{
98 int success = 1;
99 size_t controls_size = sizeof(struct op_msr) * model->num_controls; 301 size_t controls_size = sizeof(struct op_msr) * model->num_controls;
100 size_t counters_size = sizeof(struct op_msr) * model->num_counters; 302 size_t counters_size = sizeof(struct op_msr) * model->num_counters;
101 303
102 int i; 304 int i;
103 for_each_possible_cpu(i) { 305 for_each_possible_cpu(i) {
104 per_cpu(cpu_msrs, i).counters = kmalloc(counters_size, 306 per_cpu(cpu_msrs, i).counters = kmalloc(counters_size,
105 GFP_KERNEL); 307 GFP_KERNEL);
106 if (!per_cpu(cpu_msrs, i).counters) { 308 if (!per_cpu(cpu_msrs, i).counters)
107 success = 0; 309 return 0;
108 break;
109 }
110 per_cpu(cpu_msrs, i).controls = kmalloc(controls_size, 310 per_cpu(cpu_msrs, i).controls = kmalloc(controls_size,
111 GFP_KERNEL); 311 GFP_KERNEL);
112 if (!per_cpu(cpu_msrs, i).controls) { 312 if (!per_cpu(cpu_msrs, i).controls)
113 success = 0; 313 return 0;
114 break;
115 }
116 } 314 }
117 315
118 if (!success) 316 return 1;
119 free_msrs();
120
121 return success;
122} 317}
123 318
124static void nmi_cpu_setup(void *dummy) 319static void nmi_cpu_setup(void *dummy)
125{ 320{
126 int cpu = smp_processor_id(); 321 int cpu = smp_processor_id();
127 struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu); 322 struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
323 nmi_cpu_save_registers(msrs);
128 spin_lock(&oprofilefs_lock); 324 spin_lock(&oprofilefs_lock);
129 model->setup_ctrs(msrs); 325 model->setup_ctrs(model, msrs);
326 nmi_cpu_setup_mux(cpu, msrs);
130 spin_unlock(&oprofilefs_lock); 327 spin_unlock(&oprofilefs_lock);
131 per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC); 328 per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC);
132 apic_write(APIC_LVTPC, APIC_DM_NMI); 329 apic_write(APIC_LVTPC, APIC_DM_NMI);
@@ -144,11 +341,15 @@ static int nmi_setup(void)
144 int cpu; 341 int cpu;
145 342
146 if (!allocate_msrs()) 343 if (!allocate_msrs())
147 return -ENOMEM; 344 err = -ENOMEM;
345 else if (!nmi_setup_mux())
346 err = -ENOMEM;
347 else
348 err = register_die_notifier(&profile_exceptions_nb);
148 349
149 err = register_die_notifier(&profile_exceptions_nb);
150 if (err) { 350 if (err) {
151 free_msrs(); 351 free_msrs();
352 nmi_shutdown_mux();
152 return err; 353 return err;
153 } 354 }
154 355
@@ -159,45 +360,38 @@ static int nmi_setup(void)
159 /* Assume saved/restored counters are the same on all CPUs */ 360 /* Assume saved/restored counters are the same on all CPUs */
160 model->fill_in_addresses(&per_cpu(cpu_msrs, 0)); 361 model->fill_in_addresses(&per_cpu(cpu_msrs, 0));
161 for_each_possible_cpu(cpu) { 362 for_each_possible_cpu(cpu) {
162 if (cpu != 0) { 363 if (!cpu)
163 memcpy(per_cpu(cpu_msrs, cpu).counters, 364 continue;
164 per_cpu(cpu_msrs, 0).counters, 365
165 sizeof(struct op_msr) * model->num_counters); 366 memcpy(per_cpu(cpu_msrs, cpu).counters,
166 367 per_cpu(cpu_msrs, 0).counters,
167 memcpy(per_cpu(cpu_msrs, cpu).controls, 368 sizeof(struct op_msr) * model->num_counters);
168 per_cpu(cpu_msrs, 0).controls, 369
169 sizeof(struct op_msr) * model->num_controls); 370 memcpy(per_cpu(cpu_msrs, cpu).controls,
170 } 371 per_cpu(cpu_msrs, 0).controls,
372 sizeof(struct op_msr) * model->num_controls);
171 373
374 mux_clone(cpu);
172 } 375 }
173 on_each_cpu(nmi_save_registers, NULL, 1);
174 on_each_cpu(nmi_cpu_setup, NULL, 1); 376 on_each_cpu(nmi_cpu_setup, NULL, 1);
175 nmi_enabled = 1; 377 nmi_enabled = 1;
176 return 0; 378 return 0;
177} 379}
178 380
179static void nmi_restore_registers(struct op_msrs *msrs) 381static void nmi_cpu_restore_registers(struct op_msrs *msrs)
180{ 382{
181 unsigned int const nr_ctrs = model->num_counters;
182 unsigned int const nr_ctrls = model->num_controls;
183 struct op_msr *counters = msrs->counters; 383 struct op_msr *counters = msrs->counters;
184 struct op_msr *controls = msrs->controls; 384 struct op_msr *controls = msrs->controls;
185 unsigned int i; 385 unsigned int i;
186 386
187 for (i = 0; i < nr_ctrls; ++i) { 387 for (i = 0; i < model->num_controls; ++i) {
188 if (controls[i].addr) { 388 if (controls[i].addr)
189 wrmsr(controls[i].addr, 389 wrmsrl(controls[i].addr, controls[i].saved);
190 controls[i].saved.low,
191 controls[i].saved.high);
192 }
193 } 390 }
194 391
195 for (i = 0; i < nr_ctrs; ++i) { 392 for (i = 0; i < model->num_counters; ++i) {
196 if (counters[i].addr) { 393 if (counters[i].addr)
197 wrmsr(counters[i].addr, 394 wrmsrl(counters[i].addr, counters[i].saved);
198 counters[i].saved.low,
199 counters[i].saved.high);
200 }
201 } 395 }
202} 396}
203 397
@@ -205,7 +399,7 @@ static void nmi_cpu_shutdown(void *dummy)
205{ 399{
206 unsigned int v; 400 unsigned int v;
207 int cpu = smp_processor_id(); 401 int cpu = smp_processor_id();
208 struct op_msrs *msrs = &__get_cpu_var(cpu_msrs); 402 struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
209 403
210 /* restoring APIC_LVTPC can trigger an apic error because the delivery 404 /* restoring APIC_LVTPC can trigger an apic error because the delivery
211 * mode and vector nr combination can be illegal. That's by design: on 405 * mode and vector nr combination can be illegal. That's by design: on
@@ -216,7 +410,7 @@ static void nmi_cpu_shutdown(void *dummy)
216 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED); 410 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
217 apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu)); 411 apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu));
218 apic_write(APIC_LVTERR, v); 412 apic_write(APIC_LVTERR, v);
219 nmi_restore_registers(msrs); 413 nmi_cpu_restore_registers(msrs);
220} 414}
221 415
222static void nmi_shutdown(void) 416static void nmi_shutdown(void)
@@ -226,42 +420,18 @@ static void nmi_shutdown(void)
226 nmi_enabled = 0; 420 nmi_enabled = 0;
227 on_each_cpu(nmi_cpu_shutdown, NULL, 1); 421 on_each_cpu(nmi_cpu_shutdown, NULL, 1);
228 unregister_die_notifier(&profile_exceptions_nb); 422 unregister_die_notifier(&profile_exceptions_nb);
423 nmi_shutdown_mux();
229 msrs = &get_cpu_var(cpu_msrs); 424 msrs = &get_cpu_var(cpu_msrs);
230 model->shutdown(msrs); 425 model->shutdown(msrs);
231 free_msrs(); 426 free_msrs();
232 put_cpu_var(cpu_msrs); 427 put_cpu_var(cpu_msrs);
233} 428}
234 429
235static void nmi_cpu_start(void *dummy)
236{
237 struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
238 model->start(msrs);
239}
240
241static int nmi_start(void)
242{
243 on_each_cpu(nmi_cpu_start, NULL, 1);
244 return 0;
245}
246
247static void nmi_cpu_stop(void *dummy)
248{
249 struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
250 model->stop(msrs);
251}
252
253static void nmi_stop(void)
254{
255 on_each_cpu(nmi_cpu_stop, NULL, 1);
256}
257
258struct op_counter_config counter_config[OP_MAX_COUNTER];
259
260static int nmi_create_files(struct super_block *sb, struct dentry *root) 430static int nmi_create_files(struct super_block *sb, struct dentry *root)
261{ 431{
262 unsigned int i; 432 unsigned int i;
263 433
264 for (i = 0; i < model->num_counters; ++i) { 434 for (i = 0; i < model->num_virt_counters; ++i) {
265 struct dentry *dir; 435 struct dentry *dir;
266 char buf[4]; 436 char buf[4];
267 437
@@ -270,7 +440,7 @@ static int nmi_create_files(struct super_block *sb, struct dentry *root)
270 * NOTE: assumes 1:1 mapping here (that counters are organized 440 * NOTE: assumes 1:1 mapping here (that counters are organized
271 * sequentially in their struct assignment). 441 * sequentially in their struct assignment).
272 */ 442 */
273 if (unlikely(!avail_to_resrv_perfctr_nmi_bit(i))) 443 if (!avail_to_resrv_perfctr_nmi_bit(op_x86_virt_to_phys(i)))
274 continue; 444 continue;
275 445
276 snprintf(buf, sizeof(buf), "%d", i); 446 snprintf(buf, sizeof(buf), "%d", i);
@@ -402,6 +572,7 @@ module_param_call(cpu_type, force_cpu_type, NULL, NULL, 0);
402static int __init ppro_init(char **cpu_type) 572static int __init ppro_init(char **cpu_type)
403{ 573{
404 __u8 cpu_model = boot_cpu_data.x86_model; 574 __u8 cpu_model = boot_cpu_data.x86_model;
575 struct op_x86_model_spec *spec = &op_ppro_spec; /* default */
405 576
406 if (force_arch_perfmon && cpu_has_arch_perfmon) 577 if (force_arch_perfmon && cpu_has_arch_perfmon)
407 return 0; 578 return 0;
@@ -428,7 +599,7 @@ static int __init ppro_init(char **cpu_type)
428 *cpu_type = "i386/core_2"; 599 *cpu_type = "i386/core_2";
429 break; 600 break;
430 case 26: 601 case 26:
431 arch_perfmon_setup_counters(); 602 spec = &op_arch_perfmon_spec;
432 *cpu_type = "i386/core_i7"; 603 *cpu_type = "i386/core_i7";
433 break; 604 break;
434 case 28: 605 case 28:
@@ -439,17 +610,7 @@ static int __init ppro_init(char **cpu_type)
439 return 0; 610 return 0;
440 } 611 }
441 612
442 model = &op_ppro_spec; 613 model = spec;
443 return 1;
444}
445
446static int __init arch_perfmon_init(char **cpu_type)
447{
448 if (!cpu_has_arch_perfmon)
449 return 0;
450 *cpu_type = "i386/arch_perfmon";
451 model = &op_arch_perfmon_spec;
452 arch_perfmon_setup_counters();
453 return 1; 614 return 1;
454} 615}
455 616
@@ -471,27 +632,26 @@ int __init op_nmi_init(struct oprofile_operations *ops)
471 /* Needs to be at least an Athlon (or hammer in 32bit mode) */ 632 /* Needs to be at least an Athlon (or hammer in 32bit mode) */
472 633
473 switch (family) { 634 switch (family) {
474 default:
475 return -ENODEV;
476 case 6: 635 case 6:
477 model = &op_amd_spec;
478 cpu_type = "i386/athlon"; 636 cpu_type = "i386/athlon";
479 break; 637 break;
480 case 0xf: 638 case 0xf:
481 model = &op_amd_spec; 639 /*
482 /* Actually it could be i386/hammer too, but give 640 * Actually it could be i386/hammer too, but
483 user space an consistent name. */ 641 * give user space an consistent name.
642 */
484 cpu_type = "x86-64/hammer"; 643 cpu_type = "x86-64/hammer";
485 break; 644 break;
486 case 0x10: 645 case 0x10:
487 model = &op_amd_spec;
488 cpu_type = "x86-64/family10"; 646 cpu_type = "x86-64/family10";
489 break; 647 break;
490 case 0x11: 648 case 0x11:
491 model = &op_amd_spec;
492 cpu_type = "x86-64/family11h"; 649 cpu_type = "x86-64/family11h";
493 break; 650 break;
651 default:
652 return -ENODEV;
494 } 653 }
654 model = &op_amd_spec;
495 break; 655 break;
496 656
497 case X86_VENDOR_INTEL: 657 case X86_VENDOR_INTEL:
@@ -510,8 +670,15 @@ int __init op_nmi_init(struct oprofile_operations *ops)
510 break; 670 break;
511 } 671 }
512 672
513 if (!cpu_type && !arch_perfmon_init(&cpu_type)) 673 if (cpu_type)
674 break;
675
676 if (!cpu_has_arch_perfmon)
514 return -ENODEV; 677 return -ENODEV;
678
679 /* use arch perfmon as fallback */
680 cpu_type = "i386/arch_perfmon";
681 model = &op_arch_perfmon_spec;
515 break; 682 break;
516 683
517 default: 684 default:
@@ -522,18 +689,23 @@ int __init op_nmi_init(struct oprofile_operations *ops)
522 register_cpu_notifier(&oprofile_cpu_nb); 689 register_cpu_notifier(&oprofile_cpu_nb);
523#endif 690#endif
524 /* default values, can be overwritten by model */ 691 /* default values, can be overwritten by model */
525 ops->create_files = nmi_create_files; 692 ops->create_files = nmi_create_files;
526 ops->setup = nmi_setup; 693 ops->setup = nmi_setup;
527 ops->shutdown = nmi_shutdown; 694 ops->shutdown = nmi_shutdown;
528 ops->start = nmi_start; 695 ops->start = nmi_start;
529 ops->stop = nmi_stop; 696 ops->stop = nmi_stop;
530 ops->cpu_type = cpu_type; 697 ops->cpu_type = cpu_type;
531 698
532 if (model->init) 699 if (model->init)
533 ret = model->init(ops); 700 ret = model->init(ops);
534 if (ret) 701 if (ret)
535 return ret; 702 return ret;
536 703
704 if (!model->num_virt_counters)
705 model->num_virt_counters = model->num_counters;
706
707 mux_init(ops);
708
537 init_sysfs(); 709 init_sysfs();
538 using_nmi = 1; 710 using_nmi = 1;
539 printk(KERN_INFO "oprofile: using NMI interrupt.\n"); 711 printk(KERN_INFO "oprofile: using NMI interrupt.\n");
diff --git a/arch/x86/oprofile/op_counter.h b/arch/x86/oprofile/op_counter.h
index 91b6a116165e..e28398df0df2 100644
--- a/arch/x86/oprofile/op_counter.h
+++ b/arch/x86/oprofile/op_counter.h
@@ -10,7 +10,7 @@
10#ifndef OP_COUNTER_H 10#ifndef OP_COUNTER_H
11#define OP_COUNTER_H 11#define OP_COUNTER_H
12 12
13#define OP_MAX_COUNTER 8 13#define OP_MAX_COUNTER 32
14 14
15/* Per-perfctr configuration as set via 15/* Per-perfctr configuration as set via
16 * oprofilefs. 16 * oprofilefs.
diff --git a/arch/x86/oprofile/op_model_amd.c b/arch/x86/oprofile/op_model_amd.c
index 8fdf06e4edf9..39686c29f03a 100644
--- a/arch/x86/oprofile/op_model_amd.c
+++ b/arch/x86/oprofile/op_model_amd.c
@@ -9,12 +9,15 @@
9 * @author Philippe Elie 9 * @author Philippe Elie
10 * @author Graydon Hoare 10 * @author Graydon Hoare
11 * @author Robert Richter <robert.richter@amd.com> 11 * @author Robert Richter <robert.richter@amd.com>
12 * @author Barry Kasindorf 12 * @author Barry Kasindorf <barry.kasindorf@amd.com>
13 * @author Jason Yeh <jason.yeh@amd.com>
14 * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
13 */ 15 */
14 16
15#include <linux/oprofile.h> 17#include <linux/oprofile.h>
16#include <linux/device.h> 18#include <linux/device.h>
17#include <linux/pci.h> 19#include <linux/pci.h>
20#include <linux/percpu.h>
18 21
19#include <asm/ptrace.h> 22#include <asm/ptrace.h>
20#include <asm/msr.h> 23#include <asm/msr.h>
@@ -25,43 +28,36 @@
25 28
26#define NUM_COUNTERS 4 29#define NUM_COUNTERS 4
27#define NUM_CONTROLS 4 30#define NUM_CONTROLS 4
31#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
32#define NUM_VIRT_COUNTERS 32
33#define NUM_VIRT_CONTROLS 32
34#else
35#define NUM_VIRT_COUNTERS NUM_COUNTERS
36#define NUM_VIRT_CONTROLS NUM_CONTROLS
37#endif
38
39#define OP_EVENT_MASK 0x0FFF
40#define OP_CTR_OVERFLOW (1ULL<<31)
28 41
29#define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0) 42#define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
30#define CTR_READ(l, h, msrs, c) do {rdmsr(msrs->counters[(c)].addr, (l), (h)); } while (0) 43
31#define CTR_WRITE(l, msrs, c) do {wrmsr(msrs->counters[(c)].addr, -(unsigned int)(l), -1); } while (0) 44static unsigned long reset_value[NUM_VIRT_COUNTERS];
32#define CTR_OVERFLOWED(n) (!((n) & (1U<<31)))
33
34#define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)
35#define CTRL_READ(l, h, msrs, c) do {rdmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
36#define CTRL_WRITE(l, h, msrs, c) do {wrmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
37#define CTRL_SET_ACTIVE(n) (n |= (1<<22))
38#define CTRL_SET_INACTIVE(n) (n &= ~(1<<22))
39#define CTRL_CLEAR_LO(x) (x &= (1<<21))
40#define CTRL_CLEAR_HI(x) (x &= 0xfffffcf0)
41#define CTRL_SET_ENABLE(val) (val |= 1<<20)
42#define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16))
43#define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17))
44#define CTRL_SET_UM(val, m) (val |= (m << 8))
45#define CTRL_SET_EVENT_LOW(val, e) (val |= (e & 0xff))
46#define CTRL_SET_EVENT_HIGH(val, e) (val |= ((e >> 8) & 0xf))
47#define CTRL_SET_HOST_ONLY(val, h) (val |= ((h & 1) << 9))
48#define CTRL_SET_GUEST_ONLY(val, h) (val |= ((h & 1) << 8))
49
50static unsigned long reset_value[NUM_COUNTERS];
51 45
52#ifdef CONFIG_OPROFILE_IBS 46#ifdef CONFIG_OPROFILE_IBS
53 47
54/* IbsFetchCtl bits/masks */ 48/* IbsFetchCtl bits/masks */
55#define IBS_FETCH_HIGH_VALID_BIT (1UL << 17) /* bit 49 */ 49#define IBS_FETCH_RAND_EN (1ULL<<57)
56#define IBS_FETCH_HIGH_ENABLE (1UL << 16) /* bit 48 */ 50#define IBS_FETCH_VAL (1ULL<<49)
57#define IBS_FETCH_LOW_MAX_CNT_MASK 0x0000FFFFUL /* MaxCnt mask */ 51#define IBS_FETCH_ENABLE (1ULL<<48)
52#define IBS_FETCH_CNT_MASK 0xFFFF0000ULL
58 53
59/*IbsOpCtl bits */ 54/*IbsOpCtl bits */
60#define IBS_OP_LOW_VALID_BIT (1ULL<<18) /* bit 18 */ 55#define IBS_OP_CNT_CTL (1ULL<<19)
61#define IBS_OP_LOW_ENABLE (1ULL<<17) /* bit 17 */ 56#define IBS_OP_VAL (1ULL<<18)
57#define IBS_OP_ENABLE (1ULL<<17)
62 58
63#define IBS_FETCH_SIZE 6 59#define IBS_FETCH_SIZE 6
64#define IBS_OP_SIZE 12 60#define IBS_OP_SIZE 12
65 61
66static int has_ibs; /* AMD Family10h and later */ 62static int has_ibs; /* AMD Family10h and later */
67 63
@@ -78,6 +74,45 @@ static struct op_ibs_config ibs_config;
78 74
79#endif 75#endif
80 76
77#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
78
79static void op_mux_fill_in_addresses(struct op_msrs * const msrs)
80{
81 int i;
82
83 for (i = 0; i < NUM_VIRT_COUNTERS; i++) {
84 int hw_counter = op_x86_virt_to_phys(i);
85 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
86 msrs->multiplex[i].addr = MSR_K7_PERFCTR0 + hw_counter;
87 else
88 msrs->multiplex[i].addr = 0;
89 }
90}
91
92static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
93 struct op_msrs const * const msrs)
94{
95 u64 val;
96 int i;
97
98 /* enable active counters */
99 for (i = 0; i < NUM_COUNTERS; ++i) {
100 int virt = op_x86_phys_to_virt(i);
101 if (!counter_config[virt].enabled)
102 continue;
103 rdmsrl(msrs->controls[i].addr, val);
104 val &= model->reserved;
105 val |= op_x86_get_ctrl(model, &counter_config[virt]);
106 wrmsrl(msrs->controls[i].addr, val);
107 }
108}
109
110#else
111
112static inline void op_mux_fill_in_addresses(struct op_msrs * const msrs) { }
113
114#endif
115
81/* functions for op_amd_spec */ 116/* functions for op_amd_spec */
82 117
83static void op_amd_fill_in_addresses(struct op_msrs * const msrs) 118static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
@@ -97,150 +132,174 @@ static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
97 else 132 else
98 msrs->controls[i].addr = 0; 133 msrs->controls[i].addr = 0;
99 } 134 }
100}
101 135
136 op_mux_fill_in_addresses(msrs);
137}
102 138
103static void op_amd_setup_ctrs(struct op_msrs const * const msrs) 139static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
140 struct op_msrs const * const msrs)
104{ 141{
105 unsigned int low, high; 142 u64 val;
106 int i; 143 int i;
107 144
145 /* setup reset_value */
146 for (i = 0; i < NUM_VIRT_COUNTERS; ++i) {
147 if (counter_config[i].enabled)
148 reset_value[i] = counter_config[i].count;
149 else
150 reset_value[i] = 0;
151 }
152
108 /* clear all counters */ 153 /* clear all counters */
109 for (i = 0 ; i < NUM_CONTROLS; ++i) { 154 for (i = 0; i < NUM_CONTROLS; ++i) {
110 if (unlikely(!CTRL_IS_RESERVED(msrs, i))) 155 if (unlikely(!msrs->controls[i].addr))
111 continue; 156 continue;
112 CTRL_READ(low, high, msrs, i); 157 rdmsrl(msrs->controls[i].addr, val);
113 CTRL_CLEAR_LO(low); 158 val &= model->reserved;
114 CTRL_CLEAR_HI(high); 159 wrmsrl(msrs->controls[i].addr, val);
115 CTRL_WRITE(low, high, msrs, i);
116 } 160 }
117 161
118 /* avoid a false detection of ctr overflows in NMI handler */ 162 /* avoid a false detection of ctr overflows in NMI handler */
119 for (i = 0; i < NUM_COUNTERS; ++i) { 163 for (i = 0; i < NUM_COUNTERS; ++i) {
120 if (unlikely(!CTR_IS_RESERVED(msrs, i))) 164 if (unlikely(!msrs->counters[i].addr))
121 continue; 165 continue;
122 CTR_WRITE(1, msrs, i); 166 wrmsrl(msrs->counters[i].addr, -1LL);
123 } 167 }
124 168
125 /* enable active counters */ 169 /* enable active counters */
126 for (i = 0; i < NUM_COUNTERS; ++i) { 170 for (i = 0; i < NUM_COUNTERS; ++i) {
127 if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) { 171 int virt = op_x86_phys_to_virt(i);
128 reset_value[i] = counter_config[i].count; 172 if (!counter_config[virt].enabled)
173 continue;
174 if (!msrs->counters[i].addr)
175 continue;
129 176
130 CTR_WRITE(counter_config[i].count, msrs, i); 177 /* setup counter registers */
131 178 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
132 CTRL_READ(low, high, msrs, i); 179
133 CTRL_CLEAR_LO(low); 180 /* setup control registers */
134 CTRL_CLEAR_HI(high); 181 rdmsrl(msrs->controls[i].addr, val);
135 CTRL_SET_ENABLE(low); 182 val &= model->reserved;
136 CTRL_SET_USR(low, counter_config[i].user); 183 val |= op_x86_get_ctrl(model, &counter_config[virt]);
137 CTRL_SET_KERN(low, counter_config[i].kernel); 184 wrmsrl(msrs->controls[i].addr, val);
138 CTRL_SET_UM(low, counter_config[i].unit_mask);
139 CTRL_SET_EVENT_LOW(low, counter_config[i].event);
140 CTRL_SET_EVENT_HIGH(high, counter_config[i].event);
141 CTRL_SET_HOST_ONLY(high, 0);
142 CTRL_SET_GUEST_ONLY(high, 0);
143
144 CTRL_WRITE(low, high, msrs, i);
145 } else {
146 reset_value[i] = 0;
147 }
148 } 185 }
149} 186}
150 187
151#ifdef CONFIG_OPROFILE_IBS 188#ifdef CONFIG_OPROFILE_IBS
152 189
153static inline int 190static inline void
154op_amd_handle_ibs(struct pt_regs * const regs, 191op_amd_handle_ibs(struct pt_regs * const regs,
155 struct op_msrs const * const msrs) 192 struct op_msrs const * const msrs)
156{ 193{
157 u32 low, high; 194 u64 val, ctl;
158 u64 msr;
159 struct op_entry entry; 195 struct op_entry entry;
160 196
161 if (!has_ibs) 197 if (!has_ibs)
162 return 1; 198 return;
163 199
164 if (ibs_config.fetch_enabled) { 200 if (ibs_config.fetch_enabled) {
165 rdmsr(MSR_AMD64_IBSFETCHCTL, low, high); 201 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
166 if (high & IBS_FETCH_HIGH_VALID_BIT) { 202 if (ctl & IBS_FETCH_VAL) {
167 rdmsrl(MSR_AMD64_IBSFETCHLINAD, msr); 203 rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
168 oprofile_write_reserve(&entry, regs, msr, 204 oprofile_write_reserve(&entry, regs, val,
169 IBS_FETCH_CODE, IBS_FETCH_SIZE); 205 IBS_FETCH_CODE, IBS_FETCH_SIZE);
170 oprofile_add_data(&entry, (u32)msr); 206 oprofile_add_data64(&entry, val);
171 oprofile_add_data(&entry, (u32)(msr >> 32)); 207 oprofile_add_data64(&entry, ctl);
172 oprofile_add_data(&entry, low); 208 rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
173 oprofile_add_data(&entry, high); 209 oprofile_add_data64(&entry, val);
174 rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, msr);
175 oprofile_add_data(&entry, (u32)msr);
176 oprofile_add_data(&entry, (u32)(msr >> 32));
177 oprofile_write_commit(&entry); 210 oprofile_write_commit(&entry);
178 211
179 /* reenable the IRQ */ 212 /* reenable the IRQ */
180 high &= ~IBS_FETCH_HIGH_VALID_BIT; 213 ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT_MASK);
181 high |= IBS_FETCH_HIGH_ENABLE; 214 ctl |= IBS_FETCH_ENABLE;
182 low &= IBS_FETCH_LOW_MAX_CNT_MASK; 215 wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
183 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
184 } 216 }
185 } 217 }
186 218
187 if (ibs_config.op_enabled) { 219 if (ibs_config.op_enabled) {
188 rdmsr(MSR_AMD64_IBSOPCTL, low, high); 220 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
189 if (low & IBS_OP_LOW_VALID_BIT) { 221 if (ctl & IBS_OP_VAL) {
190 rdmsrl(MSR_AMD64_IBSOPRIP, msr); 222 rdmsrl(MSR_AMD64_IBSOPRIP, val);
191 oprofile_write_reserve(&entry, regs, msr, 223 oprofile_write_reserve(&entry, regs, val,
192 IBS_OP_CODE, IBS_OP_SIZE); 224 IBS_OP_CODE, IBS_OP_SIZE);
193 oprofile_add_data(&entry, (u32)msr); 225 oprofile_add_data64(&entry, val);
194 oprofile_add_data(&entry, (u32)(msr >> 32)); 226 rdmsrl(MSR_AMD64_IBSOPDATA, val);
195 rdmsrl(MSR_AMD64_IBSOPDATA, msr); 227 oprofile_add_data64(&entry, val);
196 oprofile_add_data(&entry, (u32)msr); 228 rdmsrl(MSR_AMD64_IBSOPDATA2, val);
197 oprofile_add_data(&entry, (u32)(msr >> 32)); 229 oprofile_add_data64(&entry, val);
198 rdmsrl(MSR_AMD64_IBSOPDATA2, msr); 230 rdmsrl(MSR_AMD64_IBSOPDATA3, val);
199 oprofile_add_data(&entry, (u32)msr); 231 oprofile_add_data64(&entry, val);
200 oprofile_add_data(&entry, (u32)(msr >> 32)); 232 rdmsrl(MSR_AMD64_IBSDCLINAD, val);
201 rdmsrl(MSR_AMD64_IBSOPDATA3, msr); 233 oprofile_add_data64(&entry, val);
202 oprofile_add_data(&entry, (u32)msr); 234 rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
203 oprofile_add_data(&entry, (u32)(msr >> 32)); 235 oprofile_add_data64(&entry, val);
204 rdmsrl(MSR_AMD64_IBSDCLINAD, msr);
205 oprofile_add_data(&entry, (u32)msr);
206 oprofile_add_data(&entry, (u32)(msr >> 32));
207 rdmsrl(MSR_AMD64_IBSDCPHYSAD, msr);
208 oprofile_add_data(&entry, (u32)msr);
209 oprofile_add_data(&entry, (u32)(msr >> 32));
210 oprofile_write_commit(&entry); 236 oprofile_write_commit(&entry);
211 237
212 /* reenable the IRQ */ 238 /* reenable the IRQ */
213 high = 0; 239 ctl &= ~IBS_OP_VAL & 0xFFFFFFFF;
214 low &= ~IBS_OP_LOW_VALID_BIT; 240 ctl |= IBS_OP_ENABLE;
215 low |= IBS_OP_LOW_ENABLE; 241 wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
216 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
217 } 242 }
218 } 243 }
244}
219 245
220 return 1; 246static inline void op_amd_start_ibs(void)
247{
248 u64 val;
249 if (has_ibs && ibs_config.fetch_enabled) {
250 val = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
251 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
252 val |= IBS_FETCH_ENABLE;
253 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
254 }
255
256 if (has_ibs && ibs_config.op_enabled) {
257 val = (ibs_config.max_cnt_op >> 4) & 0xFFFF;
258 val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
259 val |= IBS_OP_ENABLE;
260 wrmsrl(MSR_AMD64_IBSOPCTL, val);
261 }
262}
263
264static void op_amd_stop_ibs(void)
265{
266 if (has_ibs && ibs_config.fetch_enabled)
267 /* clear max count and enable */
268 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
269
270 if (has_ibs && ibs_config.op_enabled)
271 /* clear max count and enable */
272 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
221} 273}
222 274
275#else
276
277static inline void op_amd_handle_ibs(struct pt_regs * const regs,
278 struct op_msrs const * const msrs) { }
279static inline void op_amd_start_ibs(void) { }
280static inline void op_amd_stop_ibs(void) { }
281
223#endif 282#endif
224 283
225static int op_amd_check_ctrs(struct pt_regs * const regs, 284static int op_amd_check_ctrs(struct pt_regs * const regs,
226 struct op_msrs const * const msrs) 285 struct op_msrs const * const msrs)
227{ 286{
228 unsigned int low, high; 287 u64 val;
229 int i; 288 int i;
230 289
231 for (i = 0 ; i < NUM_COUNTERS; ++i) { 290 for (i = 0; i < NUM_COUNTERS; ++i) {
232 if (!reset_value[i]) 291 int virt = op_x86_phys_to_virt(i);
292 if (!reset_value[virt])
233 continue; 293 continue;
234 CTR_READ(low, high, msrs, i); 294 rdmsrl(msrs->counters[i].addr, val);
235 if (CTR_OVERFLOWED(low)) { 295 /* bit is clear if overflowed: */
236 oprofile_add_sample(regs, i); 296 if (val & OP_CTR_OVERFLOW)
237 CTR_WRITE(reset_value[i], msrs, i); 297 continue;
238 } 298 oprofile_add_sample(regs, virt);
299 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
239 } 300 }
240 301
241#ifdef CONFIG_OPROFILE_IBS
242 op_amd_handle_ibs(regs, msrs); 302 op_amd_handle_ibs(regs, msrs);
243#endif
244 303
245 /* See op_model_ppro.c */ 304 /* See op_model_ppro.c */
246 return 1; 305 return 1;
@@ -248,79 +307,50 @@ static int op_amd_check_ctrs(struct pt_regs * const regs,
248 307
249static void op_amd_start(struct op_msrs const * const msrs) 308static void op_amd_start(struct op_msrs const * const msrs)
250{ 309{
251 unsigned int low, high; 310 u64 val;
252 int i; 311 int i;
253 for (i = 0 ; i < NUM_COUNTERS ; ++i) {
254 if (reset_value[i]) {
255 CTRL_READ(low, high, msrs, i);
256 CTRL_SET_ACTIVE(low);
257 CTRL_WRITE(low, high, msrs, i);
258 }
259 }
260 312
261#ifdef CONFIG_OPROFILE_IBS 313 for (i = 0; i < NUM_COUNTERS; ++i) {
262 if (has_ibs && ibs_config.fetch_enabled) { 314 if (!reset_value[op_x86_phys_to_virt(i)])
263 low = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF; 315 continue;
264 high = ((ibs_config.rand_en & 0x1) << 25) /* bit 57 */ 316 rdmsrl(msrs->controls[i].addr, val);
265 + IBS_FETCH_HIGH_ENABLE; 317 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
266 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high); 318 wrmsrl(msrs->controls[i].addr, val);
267 } 319 }
268 320
269 if (has_ibs && ibs_config.op_enabled) { 321 op_amd_start_ibs();
270 low = ((ibs_config.max_cnt_op >> 4) & 0xFFFF)
271 + ((ibs_config.dispatched_ops & 0x1) << 19) /* bit 19 */
272 + IBS_OP_LOW_ENABLE;
273 high = 0;
274 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
275 }
276#endif
277} 322}
278 323
279
280static void op_amd_stop(struct op_msrs const * const msrs) 324static void op_amd_stop(struct op_msrs const * const msrs)
281{ 325{
282 unsigned int low, high; 326 u64 val;
283 int i; 327 int i;
284 328
285 /* 329 /*
286 * Subtle: stop on all counters to avoid race with setting our 330 * Subtle: stop on all counters to avoid race with setting our
287 * pm callback 331 * pm callback
288 */ 332 */
289 for (i = 0 ; i < NUM_COUNTERS ; ++i) { 333 for (i = 0; i < NUM_COUNTERS; ++i) {
290 if (!reset_value[i]) 334 if (!reset_value[op_x86_phys_to_virt(i)])
291 continue; 335 continue;
292 CTRL_READ(low, high, msrs, i); 336 rdmsrl(msrs->controls[i].addr, val);
293 CTRL_SET_INACTIVE(low); 337 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
294 CTRL_WRITE(low, high, msrs, i); 338 wrmsrl(msrs->controls[i].addr, val);
295 }
296
297#ifdef CONFIG_OPROFILE_IBS
298 if (has_ibs && ibs_config.fetch_enabled) {
299 /* clear max count and enable */
300 low = 0;
301 high = 0;
302 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
303 } 339 }
304 340
305 if (has_ibs && ibs_config.op_enabled) { 341 op_amd_stop_ibs();
306 /* clear max count and enable */
307 low = 0;
308 high = 0;
309 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
310 }
311#endif
312} 342}
313 343
314static void op_amd_shutdown(struct op_msrs const * const msrs) 344static void op_amd_shutdown(struct op_msrs const * const msrs)
315{ 345{
316 int i; 346 int i;
317 347
318 for (i = 0 ; i < NUM_COUNTERS ; ++i) { 348 for (i = 0; i < NUM_COUNTERS; ++i) {
319 if (CTR_IS_RESERVED(msrs, i)) 349 if (msrs->counters[i].addr)
320 release_perfctr_nmi(MSR_K7_PERFCTR0 + i); 350 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
321 } 351 }
322 for (i = 0 ; i < NUM_CONTROLS ; ++i) { 352 for (i = 0; i < NUM_CONTROLS; ++i) {
323 if (CTRL_IS_RESERVED(msrs, i)) 353 if (msrs->controls[i].addr)
324 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i); 354 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
325 } 355 }
326} 356}
@@ -490,15 +520,21 @@ static void op_amd_exit(void) {}
490 520
491#endif /* CONFIG_OPROFILE_IBS */ 521#endif /* CONFIG_OPROFILE_IBS */
492 522
493struct op_x86_model_spec const op_amd_spec = { 523struct op_x86_model_spec op_amd_spec = {
494 .init = op_amd_init,
495 .exit = op_amd_exit,
496 .num_counters = NUM_COUNTERS, 524 .num_counters = NUM_COUNTERS,
497 .num_controls = NUM_CONTROLS, 525 .num_controls = NUM_CONTROLS,
526 .num_virt_counters = NUM_VIRT_COUNTERS,
527 .reserved = MSR_AMD_EVENTSEL_RESERVED,
528 .event_mask = OP_EVENT_MASK,
529 .init = op_amd_init,
530 .exit = op_amd_exit,
498 .fill_in_addresses = &op_amd_fill_in_addresses, 531 .fill_in_addresses = &op_amd_fill_in_addresses,
499 .setup_ctrs = &op_amd_setup_ctrs, 532 .setup_ctrs = &op_amd_setup_ctrs,
500 .check_ctrs = &op_amd_check_ctrs, 533 .check_ctrs = &op_amd_check_ctrs,
501 .start = &op_amd_start, 534 .start = &op_amd_start,
502 .stop = &op_amd_stop, 535 .stop = &op_amd_stop,
503 .shutdown = &op_amd_shutdown 536 .shutdown = &op_amd_shutdown,
537#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
538 .switch_ctrl = &op_mux_switch_ctrl,
539#endif
504}; 540};
diff --git a/arch/x86/oprofile/op_model_p4.c b/arch/x86/oprofile/op_model_p4.c
index 819b131fd752..ac6b354becdf 100644
--- a/arch/x86/oprofile/op_model_p4.c
+++ b/arch/x86/oprofile/op_model_p4.c
@@ -32,6 +32,8 @@
32#define NUM_CCCRS_HT2 9 32#define NUM_CCCRS_HT2 9
33#define NUM_CONTROLS_HT2 (NUM_ESCRS_HT2 + NUM_CCCRS_HT2) 33#define NUM_CONTROLS_HT2 (NUM_ESCRS_HT2 + NUM_CCCRS_HT2)
34 34
35#define OP_CTR_OVERFLOW (1ULL<<31)
36
35static unsigned int num_counters = NUM_COUNTERS_NON_HT; 37static unsigned int num_counters = NUM_COUNTERS_NON_HT;
36static unsigned int num_controls = NUM_CONTROLS_NON_HT; 38static unsigned int num_controls = NUM_CONTROLS_NON_HT;
37 39
@@ -350,8 +352,6 @@ static struct p4_event_binding p4_events[NUM_EVENTS] = {
350#define ESCR_SET_OS_1(escr, os) ((escr) |= (((os) & 1) << 1)) 352#define ESCR_SET_OS_1(escr, os) ((escr) |= (((os) & 1) << 1))
351#define ESCR_SET_EVENT_SELECT(escr, sel) ((escr) |= (((sel) & 0x3f) << 25)) 353#define ESCR_SET_EVENT_SELECT(escr, sel) ((escr) |= (((sel) & 0x3f) << 25))
352#define ESCR_SET_EVENT_MASK(escr, mask) ((escr) |= (((mask) & 0xffff) << 9)) 354#define ESCR_SET_EVENT_MASK(escr, mask) ((escr) |= (((mask) & 0xffff) << 9))
353#define ESCR_READ(escr, high, ev, i) do {rdmsr(ev->bindings[(i)].escr_address, (escr), (high)); } while (0)
354#define ESCR_WRITE(escr, high, ev, i) do {wrmsr(ev->bindings[(i)].escr_address, (escr), (high)); } while (0)
355 355
356#define CCCR_RESERVED_BITS 0x38030FFF 356#define CCCR_RESERVED_BITS 0x38030FFF
357#define CCCR_CLEAR(cccr) ((cccr) &= CCCR_RESERVED_BITS) 357#define CCCR_CLEAR(cccr) ((cccr) &= CCCR_RESERVED_BITS)
@@ -361,17 +361,9 @@ static struct p4_event_binding p4_events[NUM_EVENTS] = {
361#define CCCR_SET_PMI_OVF_1(cccr) ((cccr) |= (1<<27)) 361#define CCCR_SET_PMI_OVF_1(cccr) ((cccr) |= (1<<27))
362#define CCCR_SET_ENABLE(cccr) ((cccr) |= (1<<12)) 362#define CCCR_SET_ENABLE(cccr) ((cccr) |= (1<<12))
363#define CCCR_SET_DISABLE(cccr) ((cccr) &= ~(1<<12)) 363#define CCCR_SET_DISABLE(cccr) ((cccr) &= ~(1<<12))
364#define CCCR_READ(low, high, i) do {rdmsr(p4_counters[(i)].cccr_address, (low), (high)); } while (0)
365#define CCCR_WRITE(low, high, i) do {wrmsr(p4_counters[(i)].cccr_address, (low), (high)); } while (0)
366#define CCCR_OVF_P(cccr) ((cccr) & (1U<<31)) 364#define CCCR_OVF_P(cccr) ((cccr) & (1U<<31))
367#define CCCR_CLEAR_OVF(cccr) ((cccr) &= (~(1U<<31))) 365#define CCCR_CLEAR_OVF(cccr) ((cccr) &= (~(1U<<31)))
368 366
369#define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)
370#define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0)
371#define CTR_READ(l, h, i) do {rdmsr(p4_counters[(i)].counter_address, (l), (h)); } while (0)
372#define CTR_WRITE(l, i) do {wrmsr(p4_counters[(i)].counter_address, -(u32)(l), -1); } while (0)
373#define CTR_OVERFLOW_P(ctr) (!((ctr) & 0x80000000))
374
375 367
376/* this assigns a "stagger" to the current CPU, which is used throughout 368/* this assigns a "stagger" to the current CPU, which is used throughout
377 the code in this module as an extra array offset, to select the "even" 369 the code in this module as an extra array offset, to select the "even"
@@ -515,7 +507,7 @@ static void pmc_setup_one_p4_counter(unsigned int ctr)
515 if (ev->bindings[i].virt_counter & counter_bit) { 507 if (ev->bindings[i].virt_counter & counter_bit) {
516 508
517 /* modify ESCR */ 509 /* modify ESCR */
518 ESCR_READ(escr, high, ev, i); 510 rdmsr(ev->bindings[i].escr_address, escr, high);
519 ESCR_CLEAR(escr); 511 ESCR_CLEAR(escr);
520 if (stag == 0) { 512 if (stag == 0) {
521 ESCR_SET_USR_0(escr, counter_config[ctr].user); 513 ESCR_SET_USR_0(escr, counter_config[ctr].user);
@@ -526,10 +518,11 @@ static void pmc_setup_one_p4_counter(unsigned int ctr)
526 } 518 }
527 ESCR_SET_EVENT_SELECT(escr, ev->event_select); 519 ESCR_SET_EVENT_SELECT(escr, ev->event_select);
528 ESCR_SET_EVENT_MASK(escr, counter_config[ctr].unit_mask); 520 ESCR_SET_EVENT_MASK(escr, counter_config[ctr].unit_mask);
529 ESCR_WRITE(escr, high, ev, i); 521 wrmsr(ev->bindings[i].escr_address, escr, high);
530 522
531 /* modify CCCR */ 523 /* modify CCCR */
532 CCCR_READ(cccr, high, VIRT_CTR(stag, ctr)); 524 rdmsr(p4_counters[VIRT_CTR(stag, ctr)].cccr_address,
525 cccr, high);
533 CCCR_CLEAR(cccr); 526 CCCR_CLEAR(cccr);
534 CCCR_SET_REQUIRED_BITS(cccr); 527 CCCR_SET_REQUIRED_BITS(cccr);
535 CCCR_SET_ESCR_SELECT(cccr, ev->escr_select); 528 CCCR_SET_ESCR_SELECT(cccr, ev->escr_select);
@@ -537,7 +530,8 @@ static void pmc_setup_one_p4_counter(unsigned int ctr)
537 CCCR_SET_PMI_OVF_0(cccr); 530 CCCR_SET_PMI_OVF_0(cccr);
538 else 531 else
539 CCCR_SET_PMI_OVF_1(cccr); 532 CCCR_SET_PMI_OVF_1(cccr);
540 CCCR_WRITE(cccr, high, VIRT_CTR(stag, ctr)); 533 wrmsr(p4_counters[VIRT_CTR(stag, ctr)].cccr_address,
534 cccr, high);
541 return; 535 return;
542 } 536 }
543 } 537 }
@@ -548,7 +542,8 @@ static void pmc_setup_one_p4_counter(unsigned int ctr)
548} 542}
549 543
550 544
551static void p4_setup_ctrs(struct op_msrs const * const msrs) 545static void p4_setup_ctrs(struct op_x86_model_spec const *model,
546 struct op_msrs const * const msrs)
552{ 547{
553 unsigned int i; 548 unsigned int i;
554 unsigned int low, high; 549 unsigned int low, high;
@@ -563,8 +558,8 @@ static void p4_setup_ctrs(struct op_msrs const * const msrs)
563 } 558 }
564 559
565 /* clear the cccrs we will use */ 560 /* clear the cccrs we will use */
566 for (i = 0 ; i < num_counters ; i++) { 561 for (i = 0; i < num_counters; i++) {
567 if (unlikely(!CTRL_IS_RESERVED(msrs, i))) 562 if (unlikely(!msrs->controls[i].addr))
568 continue; 563 continue;
569 rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high); 564 rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
570 CCCR_CLEAR(low); 565 CCCR_CLEAR(low);
@@ -574,17 +569,18 @@ static void p4_setup_ctrs(struct op_msrs const * const msrs)
574 569
575 /* clear all escrs (including those outside our concern) */ 570 /* clear all escrs (including those outside our concern) */
576 for (i = num_counters; i < num_controls; i++) { 571 for (i = num_counters; i < num_controls; i++) {
577 if (unlikely(!CTRL_IS_RESERVED(msrs, i))) 572 if (unlikely(!msrs->controls[i].addr))
578 continue; 573 continue;
579 wrmsr(msrs->controls[i].addr, 0, 0); 574 wrmsr(msrs->controls[i].addr, 0, 0);
580 } 575 }
581 576
582 /* setup all counters */ 577 /* setup all counters */
583 for (i = 0 ; i < num_counters ; ++i) { 578 for (i = 0; i < num_counters; ++i) {
584 if ((counter_config[i].enabled) && (CTRL_IS_RESERVED(msrs, i))) { 579 if (counter_config[i].enabled && msrs->controls[i].addr) {
585 reset_value[i] = counter_config[i].count; 580 reset_value[i] = counter_config[i].count;
586 pmc_setup_one_p4_counter(i); 581 pmc_setup_one_p4_counter(i);
587 CTR_WRITE(counter_config[i].count, VIRT_CTR(stag, i)); 582 wrmsrl(p4_counters[VIRT_CTR(stag, i)].counter_address,
583 -(u64)counter_config[i].count);
588 } else { 584 } else {
589 reset_value[i] = 0; 585 reset_value[i] = 0;
590 } 586 }
@@ -624,14 +620,16 @@ static int p4_check_ctrs(struct pt_regs * const regs,
624 620
625 real = VIRT_CTR(stag, i); 621 real = VIRT_CTR(stag, i);
626 622
627 CCCR_READ(low, high, real); 623 rdmsr(p4_counters[real].cccr_address, low, high);
628 CTR_READ(ctr, high, real); 624 rdmsr(p4_counters[real].counter_address, ctr, high);
629 if (CCCR_OVF_P(low) || CTR_OVERFLOW_P(ctr)) { 625 if (CCCR_OVF_P(low) || !(ctr & OP_CTR_OVERFLOW)) {
630 oprofile_add_sample(regs, i); 626 oprofile_add_sample(regs, i);
631 CTR_WRITE(reset_value[i], real); 627 wrmsrl(p4_counters[real].counter_address,
628 -(u64)reset_value[i]);
632 CCCR_CLEAR_OVF(low); 629 CCCR_CLEAR_OVF(low);
633 CCCR_WRITE(low, high, real); 630 wrmsr(p4_counters[real].cccr_address, low, high);
634 CTR_WRITE(reset_value[i], real); 631 wrmsrl(p4_counters[real].counter_address,
632 -(u64)reset_value[i]);
635 } 633 }
636 } 634 }
637 635
@@ -653,9 +651,9 @@ static void p4_start(struct op_msrs const * const msrs)
653 for (i = 0; i < num_counters; ++i) { 651 for (i = 0; i < num_counters; ++i) {
654 if (!reset_value[i]) 652 if (!reset_value[i])
655 continue; 653 continue;
656 CCCR_READ(low, high, VIRT_CTR(stag, i)); 654 rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
657 CCCR_SET_ENABLE(low); 655 CCCR_SET_ENABLE(low);
658 CCCR_WRITE(low, high, VIRT_CTR(stag, i)); 656 wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
659 } 657 }
660} 658}
661 659
@@ -670,9 +668,9 @@ static void p4_stop(struct op_msrs const * const msrs)
670 for (i = 0; i < num_counters; ++i) { 668 for (i = 0; i < num_counters; ++i) {
671 if (!reset_value[i]) 669 if (!reset_value[i])
672 continue; 670 continue;
673 CCCR_READ(low, high, VIRT_CTR(stag, i)); 671 rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
674 CCCR_SET_DISABLE(low); 672 CCCR_SET_DISABLE(low);
675 CCCR_WRITE(low, high, VIRT_CTR(stag, i)); 673 wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
676 } 674 }
677} 675}
678 676
@@ -680,8 +678,8 @@ static void p4_shutdown(struct op_msrs const * const msrs)
680{ 678{
681 int i; 679 int i;
682 680
683 for (i = 0 ; i < num_counters ; ++i) { 681 for (i = 0; i < num_counters; ++i) {
684 if (CTR_IS_RESERVED(msrs, i)) 682 if (msrs->counters[i].addr)
685 release_perfctr_nmi(msrs->counters[i].addr); 683 release_perfctr_nmi(msrs->counters[i].addr);
686 } 684 }
687 /* 685 /*
@@ -689,15 +687,15 @@ static void p4_shutdown(struct op_msrs const * const msrs)
689 * conjunction with the counter registers (hence the starting offset). 687 * conjunction with the counter registers (hence the starting offset).
690 * This saves a few bits. 688 * This saves a few bits.
691 */ 689 */
692 for (i = num_counters ; i < num_controls ; ++i) { 690 for (i = num_counters; i < num_controls; ++i) {
693 if (CTRL_IS_RESERVED(msrs, i)) 691 if (msrs->controls[i].addr)
694 release_evntsel_nmi(msrs->controls[i].addr); 692 release_evntsel_nmi(msrs->controls[i].addr);
695 } 693 }
696} 694}
697 695
698 696
699#ifdef CONFIG_SMP 697#ifdef CONFIG_SMP
700struct op_x86_model_spec const op_p4_ht2_spec = { 698struct op_x86_model_spec op_p4_ht2_spec = {
701 .num_counters = NUM_COUNTERS_HT2, 699 .num_counters = NUM_COUNTERS_HT2,
702 .num_controls = NUM_CONTROLS_HT2, 700 .num_controls = NUM_CONTROLS_HT2,
703 .fill_in_addresses = &p4_fill_in_addresses, 701 .fill_in_addresses = &p4_fill_in_addresses,
@@ -709,7 +707,7 @@ struct op_x86_model_spec const op_p4_ht2_spec = {
709}; 707};
710#endif 708#endif
711 709
712struct op_x86_model_spec const op_p4_spec = { 710struct op_x86_model_spec op_p4_spec = {
713 .num_counters = NUM_COUNTERS_NON_HT, 711 .num_counters = NUM_COUNTERS_NON_HT,
714 .num_controls = NUM_CONTROLS_NON_HT, 712 .num_controls = NUM_CONTROLS_NON_HT,
715 .fill_in_addresses = &p4_fill_in_addresses, 713 .fill_in_addresses = &p4_fill_in_addresses,
diff --git a/arch/x86/oprofile/op_model_ppro.c b/arch/x86/oprofile/op_model_ppro.c
index 4da7230b3d17..4899215999de 100644
--- a/arch/x86/oprofile/op_model_ppro.c
+++ b/arch/x86/oprofile/op_model_ppro.c
@@ -10,6 +10,7 @@
10 * @author Philippe Elie 10 * @author Philippe Elie
11 * @author Graydon Hoare 11 * @author Graydon Hoare
12 * @author Andi Kleen 12 * @author Andi Kleen
13 * @author Robert Richter <robert.richter@amd.com>
13 */ 14 */
14 15
15#include <linux/oprofile.h> 16#include <linux/oprofile.h>
@@ -18,7 +19,6 @@
18#include <asm/msr.h> 19#include <asm/msr.h>
19#include <asm/apic.h> 20#include <asm/apic.h>
20#include <asm/nmi.h> 21#include <asm/nmi.h>
21#include <asm/perf_counter.h>
22 22
23#include "op_x86_model.h" 23#include "op_x86_model.h"
24#include "op_counter.h" 24#include "op_counter.h"
@@ -26,20 +26,7 @@
26static int num_counters = 2; 26static int num_counters = 2;
27static int counter_width = 32; 27static int counter_width = 32;
28 28
29#define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0) 29#define MSR_PPRO_EVENTSEL_RESERVED ((0xFFFFFFFFULL<<32)|(1ULL<<21))
30#define CTR_OVERFLOWED(n) (!((n) & (1ULL<<(counter_width-1))))
31
32#define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)
33#define CTRL_READ(l, h, msrs, c) do {rdmsr((msrs->controls[(c)].addr), (l), (h)); } while (0)
34#define CTRL_WRITE(l, h, msrs, c) do {wrmsr((msrs->controls[(c)].addr), (l), (h)); } while (0)
35#define CTRL_SET_ACTIVE(n) (n |= (1<<22))
36#define CTRL_SET_INACTIVE(n) (n &= ~(1<<22))
37#define CTRL_CLEAR(x) (x &= (1<<21))
38#define CTRL_SET_ENABLE(val) (val |= 1<<20)
39#define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16))
40#define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17))
41#define CTRL_SET_UM(val, m) (val |= (m << 8))
42#define CTRL_SET_EVENT(val, e) (val |= e)
43 30
44static u64 *reset_value; 31static u64 *reset_value;
45 32
@@ -63,9 +50,10 @@ static void ppro_fill_in_addresses(struct op_msrs * const msrs)
63} 50}
64 51
65 52
66static void ppro_setup_ctrs(struct op_msrs const * const msrs) 53static void ppro_setup_ctrs(struct op_x86_model_spec const *model,
54 struct op_msrs const * const msrs)
67{ 55{
68 unsigned int low, high; 56 u64 val;
69 int i; 57 int i;
70 58
71 if (!reset_value) { 59 if (!reset_value) {
@@ -93,36 +81,30 @@ static void ppro_setup_ctrs(struct op_msrs const * const msrs)
93 } 81 }
94 82
95 /* clear all counters */ 83 /* clear all counters */
96 for (i = 0 ; i < num_counters; ++i) { 84 for (i = 0; i < num_counters; ++i) {
97 if (unlikely(!CTRL_IS_RESERVED(msrs, i))) 85 if (unlikely(!msrs->controls[i].addr))
98 continue; 86 continue;
99 CTRL_READ(low, high, msrs, i); 87 rdmsrl(msrs->controls[i].addr, val);
100 CTRL_CLEAR(low); 88 val &= model->reserved;
101 CTRL_WRITE(low, high, msrs, i); 89 wrmsrl(msrs->controls[i].addr, val);
102 } 90 }
103 91
104 /* avoid a false detection of ctr overflows in NMI handler */ 92 /* avoid a false detection of ctr overflows in NMI handler */
105 for (i = 0; i < num_counters; ++i) { 93 for (i = 0; i < num_counters; ++i) {
106 if (unlikely(!CTR_IS_RESERVED(msrs, i))) 94 if (unlikely(!msrs->counters[i].addr))
107 continue; 95 continue;
108 wrmsrl(msrs->counters[i].addr, -1LL); 96 wrmsrl(msrs->counters[i].addr, -1LL);
109 } 97 }
110 98
111 /* enable active counters */ 99 /* enable active counters */
112 for (i = 0; i < num_counters; ++i) { 100 for (i = 0; i < num_counters; ++i) {
113 if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) { 101 if (counter_config[i].enabled && msrs->counters[i].addr) {
114 reset_value[i] = counter_config[i].count; 102 reset_value[i] = counter_config[i].count;
115
116 wrmsrl(msrs->counters[i].addr, -reset_value[i]); 103 wrmsrl(msrs->counters[i].addr, -reset_value[i]);
117 104 rdmsrl(msrs->controls[i].addr, val);
118 CTRL_READ(low, high, msrs, i); 105 val &= model->reserved;
119 CTRL_CLEAR(low); 106 val |= op_x86_get_ctrl(model, &counter_config[i]);
120 CTRL_SET_ENABLE(low); 107 wrmsrl(msrs->controls[i].addr, val);
121 CTRL_SET_USR(low, counter_config[i].user);
122 CTRL_SET_KERN(low, counter_config[i].kernel);
123 CTRL_SET_UM(low, counter_config[i].unit_mask);
124 CTRL_SET_EVENT(low, counter_config[i].event);
125 CTRL_WRITE(low, high, msrs, i);
126 } else { 108 } else {
127 reset_value[i] = 0; 109 reset_value[i] = 0;
128 } 110 }
@@ -143,14 +125,14 @@ static int ppro_check_ctrs(struct pt_regs * const regs,
143 if (unlikely(!reset_value)) 125 if (unlikely(!reset_value))
144 goto out; 126 goto out;
145 127
146 for (i = 0 ; i < num_counters; ++i) { 128 for (i = 0; i < num_counters; ++i) {
147 if (!reset_value[i]) 129 if (!reset_value[i])
148 continue; 130 continue;
149 rdmsrl(msrs->counters[i].addr, val); 131 rdmsrl(msrs->counters[i].addr, val);
150 if (CTR_OVERFLOWED(val)) { 132 if (val & (1ULL << (counter_width - 1)))
151 oprofile_add_sample(regs, i); 133 continue;
152 wrmsrl(msrs->counters[i].addr, -reset_value[i]); 134 oprofile_add_sample(regs, i);
153 } 135 wrmsrl(msrs->counters[i].addr, -reset_value[i]);
154 } 136 }
155 137
156out: 138out:
@@ -171,16 +153,16 @@ out:
171 153
172static void ppro_start(struct op_msrs const * const msrs) 154static void ppro_start(struct op_msrs const * const msrs)
173{ 155{
174 unsigned int low, high; 156 u64 val;
175 int i; 157 int i;
176 158
177 if (!reset_value) 159 if (!reset_value)
178 return; 160 return;
179 for (i = 0; i < num_counters; ++i) { 161 for (i = 0; i < num_counters; ++i) {
180 if (reset_value[i]) { 162 if (reset_value[i]) {
181 CTRL_READ(low, high, msrs, i); 163 rdmsrl(msrs->controls[i].addr, val);
182 CTRL_SET_ACTIVE(low); 164 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
183 CTRL_WRITE(low, high, msrs, i); 165 wrmsrl(msrs->controls[i].addr, val);
184 } 166 }
185 } 167 }
186} 168}
@@ -188,7 +170,7 @@ static void ppro_start(struct op_msrs const * const msrs)
188 170
189static void ppro_stop(struct op_msrs const * const msrs) 171static void ppro_stop(struct op_msrs const * const msrs)
190{ 172{
191 unsigned int low, high; 173 u64 val;
192 int i; 174 int i;
193 175
194 if (!reset_value) 176 if (!reset_value)
@@ -196,9 +178,9 @@ static void ppro_stop(struct op_msrs const * const msrs)
196 for (i = 0; i < num_counters; ++i) { 178 for (i = 0; i < num_counters; ++i) {
197 if (!reset_value[i]) 179 if (!reset_value[i])
198 continue; 180 continue;
199 CTRL_READ(low, high, msrs, i); 181 rdmsrl(msrs->controls[i].addr, val);
200 CTRL_SET_INACTIVE(low); 182 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
201 CTRL_WRITE(low, high, msrs, i); 183 wrmsrl(msrs->controls[i].addr, val);
202 } 184 }
203} 185}
204 186
@@ -206,12 +188,12 @@ static void ppro_shutdown(struct op_msrs const * const msrs)
206{ 188{
207 int i; 189 int i;
208 190
209 for (i = 0 ; i < num_counters ; ++i) { 191 for (i = 0; i < num_counters; ++i) {
210 if (CTR_IS_RESERVED(msrs, i)) 192 if (msrs->counters[i].addr)
211 release_perfctr_nmi(MSR_P6_PERFCTR0 + i); 193 release_perfctr_nmi(MSR_P6_PERFCTR0 + i);
212 } 194 }
213 for (i = 0 ; i < num_counters ; ++i) { 195 for (i = 0; i < num_counters; ++i) {
214 if (CTRL_IS_RESERVED(msrs, i)) 196 if (msrs->controls[i].addr)
215 release_evntsel_nmi(MSR_P6_EVNTSEL0 + i); 197 release_evntsel_nmi(MSR_P6_EVNTSEL0 + i);
216 } 198 }
217 if (reset_value) { 199 if (reset_value) {
@@ -222,8 +204,9 @@ static void ppro_shutdown(struct op_msrs const * const msrs)
222 204
223 205
224struct op_x86_model_spec op_ppro_spec = { 206struct op_x86_model_spec op_ppro_spec = {
225 .num_counters = 2, /* can be overriden */ 207 .num_counters = 2,
226 .num_controls = 2, /* dito */ 208 .num_controls = 2,
209 .reserved = MSR_PPRO_EVENTSEL_RESERVED,
227 .fill_in_addresses = &ppro_fill_in_addresses, 210 .fill_in_addresses = &ppro_fill_in_addresses,
228 .setup_ctrs = &ppro_setup_ctrs, 211 .setup_ctrs = &ppro_setup_ctrs,
229 .check_ctrs = &ppro_check_ctrs, 212 .check_ctrs = &ppro_check_ctrs,
@@ -241,7 +224,7 @@ struct op_x86_model_spec op_ppro_spec = {
241 * the specific CPU. 224 * the specific CPU.
242 */ 225 */
243 226
244void arch_perfmon_setup_counters(void) 227static void arch_perfmon_setup_counters(void)
245{ 228{
246 union cpuid10_eax eax; 229 union cpuid10_eax eax;
247 230
@@ -259,11 +242,17 @@ void arch_perfmon_setup_counters(void)
259 242
260 op_arch_perfmon_spec.num_counters = num_counters; 243 op_arch_perfmon_spec.num_counters = num_counters;
261 op_arch_perfmon_spec.num_controls = num_counters; 244 op_arch_perfmon_spec.num_controls = num_counters;
262 op_ppro_spec.num_counters = num_counters; 245}
263 op_ppro_spec.num_controls = num_counters; 246
247static int arch_perfmon_init(struct oprofile_operations *ignore)
248{
249 arch_perfmon_setup_counters();
250 return 0;
264} 251}
265 252
266struct op_x86_model_spec op_arch_perfmon_spec = { 253struct op_x86_model_spec op_arch_perfmon_spec = {
254 .reserved = MSR_PPRO_EVENTSEL_RESERVED,
255 .init = &arch_perfmon_init,
267 /* num_counters/num_controls filled in at runtime */ 256 /* num_counters/num_controls filled in at runtime */
268 .fill_in_addresses = &ppro_fill_in_addresses, 257 .fill_in_addresses = &ppro_fill_in_addresses,
269 /* user space does the cpuid check for available events */ 258 /* user space does the cpuid check for available events */
diff --git a/arch/x86/oprofile/op_x86_model.h b/arch/x86/oprofile/op_x86_model.h
index 825e79064d64..b83776180c7f 100644
--- a/arch/x86/oprofile/op_x86_model.h
+++ b/arch/x86/oprofile/op_x86_model.h
@@ -6,51 +6,66 @@
6 * @remark Read the file COPYING 6 * @remark Read the file COPYING
7 * 7 *
8 * @author Graydon Hoare 8 * @author Graydon Hoare
9 * @author Robert Richter <robert.richter@amd.com>
9 */ 10 */
10 11
11#ifndef OP_X86_MODEL_H 12#ifndef OP_X86_MODEL_H
12#define OP_X86_MODEL_H 13#define OP_X86_MODEL_H
13 14
14struct op_saved_msr { 15#include <asm/types.h>
15 unsigned int high; 16#include <asm/perf_counter.h>
16 unsigned int low;
17};
18 17
19struct op_msr { 18struct op_msr {
20 unsigned long addr; 19 unsigned long addr;
21 struct op_saved_msr saved; 20 u64 saved;
22}; 21};
23 22
24struct op_msrs { 23struct op_msrs {
25 struct op_msr *counters; 24 struct op_msr *counters;
26 struct op_msr *controls; 25 struct op_msr *controls;
26 struct op_msr *multiplex;
27}; 27};
28 28
29struct pt_regs; 29struct pt_regs;
30 30
31struct oprofile_operations;
32
31/* The model vtable abstracts the differences between 33/* The model vtable abstracts the differences between
32 * various x86 CPU models' perfctr support. 34 * various x86 CPU models' perfctr support.
33 */ 35 */
34struct op_x86_model_spec { 36struct op_x86_model_spec {
35 int (*init)(struct oprofile_operations *ops); 37 unsigned int num_counters;
36 void (*exit)(void); 38 unsigned int num_controls;
37 unsigned int num_counters; 39 unsigned int num_virt_counters;
38 unsigned int num_controls; 40 u64 reserved;
39 void (*fill_in_addresses)(struct op_msrs * const msrs); 41 u16 event_mask;
40 void (*setup_ctrs)(struct op_msrs const * const msrs); 42 int (*init)(struct oprofile_operations *ops);
41 int (*check_ctrs)(struct pt_regs * const regs, 43 void (*exit)(void);
42 struct op_msrs const * const msrs); 44 void (*fill_in_addresses)(struct op_msrs * const msrs);
43 void (*start)(struct op_msrs const * const msrs); 45 void (*setup_ctrs)(struct op_x86_model_spec const *model,
44 void (*stop)(struct op_msrs const * const msrs); 46 struct op_msrs const * const msrs);
45 void (*shutdown)(struct op_msrs const * const msrs); 47 int (*check_ctrs)(struct pt_regs * const regs,
48 struct op_msrs const * const msrs);
49 void (*start)(struct op_msrs const * const msrs);
50 void (*stop)(struct op_msrs const * const msrs);
51 void (*shutdown)(struct op_msrs const * const msrs);
52#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
53 void (*switch_ctrl)(struct op_x86_model_spec const *model,
54 struct op_msrs const * const msrs);
55#endif
46}; 56};
47 57
58struct op_counter_config;
59
60extern u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
61 struct op_counter_config *counter_config);
62extern int op_x86_phys_to_virt(int phys);
63extern int op_x86_virt_to_phys(int virt);
64
48extern struct op_x86_model_spec op_ppro_spec; 65extern struct op_x86_model_spec op_ppro_spec;
49extern struct op_x86_model_spec const op_p4_spec; 66extern struct op_x86_model_spec op_p4_spec;
50extern struct op_x86_model_spec const op_p4_ht2_spec; 67extern struct op_x86_model_spec op_p4_ht2_spec;
51extern struct op_x86_model_spec const op_amd_spec; 68extern struct op_x86_model_spec op_amd_spec;
52extern struct op_x86_model_spec op_arch_perfmon_spec; 69extern struct op_x86_model_spec op_arch_perfmon_spec;
53 70
54extern void arch_perfmon_setup_counters(void);
55
56#endif /* OP_X86_MODEL_H */ 71#endif /* OP_X86_MODEL_H */
diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c
index 242257b19441..a7aae24f2889 100644
--- a/drivers/oprofile/cpu_buffer.c
+++ b/drivers/oprofile/cpu_buffer.c
@@ -21,7 +21,6 @@
21 21
22#include <linux/sched.h> 22#include <linux/sched.h>
23#include <linux/oprofile.h> 23#include <linux/oprofile.h>
24#include <linux/vmalloc.h>
25#include <linux/errno.h> 24#include <linux/errno.h>
26 25
27#include "event_buffer.h" 26#include "event_buffer.h"
@@ -407,6 +406,21 @@ int oprofile_add_data(struct op_entry *entry, unsigned long val)
407 return op_cpu_buffer_add_data(entry, val); 406 return op_cpu_buffer_add_data(entry, val);
408} 407}
409 408
409int oprofile_add_data64(struct op_entry *entry, u64 val)
410{
411 if (!entry->event)
412 return 0;
413 if (op_cpu_buffer_get_size(entry) < 2)
414 /*
415 * the function returns 0 to indicate a too small
416 * buffer, even if there is some space left
417 */
418 return 0;
419 if (!op_cpu_buffer_add_data(entry, (u32)val))
420 return 0;
421 return op_cpu_buffer_add_data(entry, (u32)(val >> 32));
422}
423
410int oprofile_write_commit(struct op_entry *entry) 424int oprofile_write_commit(struct op_entry *entry)
411{ 425{
412 if (!entry->event) 426 if (!entry->event)
diff --git a/drivers/oprofile/oprof.c b/drivers/oprofile/oprof.c
index 3cffce90f82a..dc8a0428260d 100644
--- a/drivers/oprofile/oprof.c
+++ b/drivers/oprofile/oprof.c
@@ -12,6 +12,8 @@
12#include <linux/init.h> 12#include <linux/init.h>
13#include <linux/oprofile.h> 13#include <linux/oprofile.h>
14#include <linux/moduleparam.h> 14#include <linux/moduleparam.h>
15#include <linux/workqueue.h>
16#include <linux/time.h>
15#include <asm/mutex.h> 17#include <asm/mutex.h>
16 18
17#include "oprof.h" 19#include "oprof.h"
@@ -87,6 +89,69 @@ out:
87 return err; 89 return err;
88} 90}
89 91
92#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
93
94static void switch_worker(struct work_struct *work);
95static DECLARE_DELAYED_WORK(switch_work, switch_worker);
96
97static void start_switch_worker(void)
98{
99 if (oprofile_ops.switch_events)
100 schedule_delayed_work(&switch_work, oprofile_time_slice);
101}
102
103static void stop_switch_worker(void)
104{
105 cancel_delayed_work_sync(&switch_work);
106}
107
108static void switch_worker(struct work_struct *work)
109{
110 if (oprofile_ops.switch_events())
111 return;
112
113 atomic_inc(&oprofile_stats.multiplex_counter);
114 start_switch_worker();
115}
116
117/* User inputs in ms, converts to jiffies */
118int oprofile_set_timeout(unsigned long val_msec)
119{
120 int err = 0;
121 unsigned long time_slice;
122
123 mutex_lock(&start_mutex);
124
125 if (oprofile_started) {
126 err = -EBUSY;
127 goto out;
128 }
129
130 if (!oprofile_ops.switch_events) {
131 err = -EINVAL;
132 goto out;
133 }
134
135 time_slice = msecs_to_jiffies(val_msec);
136 if (time_slice == MAX_JIFFY_OFFSET) {
137 err = -EINVAL;
138 goto out;
139 }
140
141 oprofile_time_slice = time_slice;
142
143out:
144 mutex_unlock(&start_mutex);
145 return err;
146
147}
148
149#else
150
151static inline void start_switch_worker(void) { }
152static inline void stop_switch_worker(void) { }
153
154#endif
90 155
91/* Actually start profiling (echo 1>/dev/oprofile/enable) */ 156/* Actually start profiling (echo 1>/dev/oprofile/enable) */
92int oprofile_start(void) 157int oprofile_start(void)
@@ -108,6 +173,8 @@ int oprofile_start(void)
108 if ((err = oprofile_ops.start())) 173 if ((err = oprofile_ops.start()))
109 goto out; 174 goto out;
110 175
176 start_switch_worker();
177
111 oprofile_started = 1; 178 oprofile_started = 1;
112out: 179out:
113 mutex_unlock(&start_mutex); 180 mutex_unlock(&start_mutex);
@@ -123,6 +190,9 @@ void oprofile_stop(void)
123 goto out; 190 goto out;
124 oprofile_ops.stop(); 191 oprofile_ops.stop();
125 oprofile_started = 0; 192 oprofile_started = 0;
193
194 stop_switch_worker();
195
126 /* wake up the daemon to read what remains */ 196 /* wake up the daemon to read what remains */
127 wake_up_buffer_waiter(); 197 wake_up_buffer_waiter();
128out: 198out:
@@ -155,7 +225,6 @@ post_sync:
155 mutex_unlock(&start_mutex); 225 mutex_unlock(&start_mutex);
156} 226}
157 227
158
159int oprofile_set_backtrace(unsigned long val) 228int oprofile_set_backtrace(unsigned long val)
160{ 229{
161 int err = 0; 230 int err = 0;
diff --git a/drivers/oprofile/oprof.h b/drivers/oprofile/oprof.h
index c288d3c24b50..cb92f5c98c1a 100644
--- a/drivers/oprofile/oprof.h
+++ b/drivers/oprofile/oprof.h
@@ -24,6 +24,8 @@ struct oprofile_operations;
24extern unsigned long oprofile_buffer_size; 24extern unsigned long oprofile_buffer_size;
25extern unsigned long oprofile_cpu_buffer_size; 25extern unsigned long oprofile_cpu_buffer_size;
26extern unsigned long oprofile_buffer_watershed; 26extern unsigned long oprofile_buffer_watershed;
27extern unsigned long oprofile_time_slice;
28
27extern struct oprofile_operations oprofile_ops; 29extern struct oprofile_operations oprofile_ops;
28extern unsigned long oprofile_started; 30extern unsigned long oprofile_started;
29extern unsigned long oprofile_backtrace_depth; 31extern unsigned long oprofile_backtrace_depth;
@@ -35,5 +37,6 @@ void oprofile_create_files(struct super_block *sb, struct dentry *root);
35void oprofile_timer_init(struct oprofile_operations *ops); 37void oprofile_timer_init(struct oprofile_operations *ops);
36 38
37int oprofile_set_backtrace(unsigned long depth); 39int oprofile_set_backtrace(unsigned long depth);
40int oprofile_set_timeout(unsigned long time);
38 41
39#endif /* OPROF_H */ 42#endif /* OPROF_H */
diff --git a/drivers/oprofile/oprofile_files.c b/drivers/oprofile/oprofile_files.c
index 5d36ffc30dd5..bbd7516e0869 100644
--- a/drivers/oprofile/oprofile_files.c
+++ b/drivers/oprofile/oprofile_files.c
@@ -9,6 +9,7 @@
9 9
10#include <linux/fs.h> 10#include <linux/fs.h>
11#include <linux/oprofile.h> 11#include <linux/oprofile.h>
12#include <linux/jiffies.h>
12 13
13#include "event_buffer.h" 14#include "event_buffer.h"
14#include "oprofile_stats.h" 15#include "oprofile_stats.h"
@@ -17,10 +18,51 @@
17#define BUFFER_SIZE_DEFAULT 131072 18#define BUFFER_SIZE_DEFAULT 131072
18#define CPU_BUFFER_SIZE_DEFAULT 8192 19#define CPU_BUFFER_SIZE_DEFAULT 8192
19#define BUFFER_WATERSHED_DEFAULT 32768 /* FIXME: tune */ 20#define BUFFER_WATERSHED_DEFAULT 32768 /* FIXME: tune */
21#define TIME_SLICE_DEFAULT 1
20 22
21unsigned long oprofile_buffer_size; 23unsigned long oprofile_buffer_size;
22unsigned long oprofile_cpu_buffer_size; 24unsigned long oprofile_cpu_buffer_size;
23unsigned long oprofile_buffer_watershed; 25unsigned long oprofile_buffer_watershed;
26unsigned long oprofile_time_slice;
27
28#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
29
30static ssize_t timeout_read(struct file *file, char __user *buf,
31 size_t count, loff_t *offset)
32{
33 return oprofilefs_ulong_to_user(jiffies_to_msecs(oprofile_time_slice),
34 buf, count, offset);
35}
36
37
38static ssize_t timeout_write(struct file *file, char const __user *buf,
39 size_t count, loff_t *offset)
40{
41 unsigned long val;
42 int retval;
43
44 if (*offset)
45 return -EINVAL;
46
47 retval = oprofilefs_ulong_from_user(&val, buf, count);
48 if (retval)
49 return retval;
50
51 retval = oprofile_set_timeout(val);
52
53 if (retval)
54 return retval;
55 return count;
56}
57
58
59static const struct file_operations timeout_fops = {
60 .read = timeout_read,
61 .write = timeout_write,
62};
63
64#endif
65
24 66
25static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) 67static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
26{ 68{
@@ -129,6 +171,7 @@ void oprofile_create_files(struct super_block *sb, struct dentry *root)
129 oprofile_buffer_size = BUFFER_SIZE_DEFAULT; 171 oprofile_buffer_size = BUFFER_SIZE_DEFAULT;
130 oprofile_cpu_buffer_size = CPU_BUFFER_SIZE_DEFAULT; 172 oprofile_cpu_buffer_size = CPU_BUFFER_SIZE_DEFAULT;
131 oprofile_buffer_watershed = BUFFER_WATERSHED_DEFAULT; 173 oprofile_buffer_watershed = BUFFER_WATERSHED_DEFAULT;
174 oprofile_time_slice = msecs_to_jiffies(TIME_SLICE_DEFAULT);
132 175
133 oprofilefs_create_file(sb, root, "enable", &enable_fops); 176 oprofilefs_create_file(sb, root, "enable", &enable_fops);
134 oprofilefs_create_file_perm(sb, root, "dump", &dump_fops, 0666); 177 oprofilefs_create_file_perm(sb, root, "dump", &dump_fops, 0666);
@@ -139,6 +182,9 @@ void oprofile_create_files(struct super_block *sb, struct dentry *root)
139 oprofilefs_create_file(sb, root, "cpu_type", &cpu_type_fops); 182 oprofilefs_create_file(sb, root, "cpu_type", &cpu_type_fops);
140 oprofilefs_create_file(sb, root, "backtrace_depth", &depth_fops); 183 oprofilefs_create_file(sb, root, "backtrace_depth", &depth_fops);
141 oprofilefs_create_file(sb, root, "pointer_size", &pointer_size_fops); 184 oprofilefs_create_file(sb, root, "pointer_size", &pointer_size_fops);
185#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
186 oprofilefs_create_file(sb, root, "time_slice", &timeout_fops);
187#endif
142 oprofile_create_stats_files(sb, root); 188 oprofile_create_stats_files(sb, root);
143 if (oprofile_ops.create_files) 189 if (oprofile_ops.create_files)
144 oprofile_ops.create_files(sb, root); 190 oprofile_ops.create_files(sb, root);
diff --git a/drivers/oprofile/oprofile_stats.c b/drivers/oprofile/oprofile_stats.c
index 3c2270a8300c..61689e814d46 100644
--- a/drivers/oprofile/oprofile_stats.c
+++ b/drivers/oprofile/oprofile_stats.c
@@ -34,6 +34,7 @@ void oprofile_reset_stats(void)
34 atomic_set(&oprofile_stats.sample_lost_no_mapping, 0); 34 atomic_set(&oprofile_stats.sample_lost_no_mapping, 0);
35 atomic_set(&oprofile_stats.event_lost_overflow, 0); 35 atomic_set(&oprofile_stats.event_lost_overflow, 0);
36 atomic_set(&oprofile_stats.bt_lost_no_mapping, 0); 36 atomic_set(&oprofile_stats.bt_lost_no_mapping, 0);
37 atomic_set(&oprofile_stats.multiplex_counter, 0);
37} 38}
38 39
39 40
@@ -76,4 +77,8 @@ void oprofile_create_stats_files(struct super_block *sb, struct dentry *root)
76 &oprofile_stats.event_lost_overflow); 77 &oprofile_stats.event_lost_overflow);
77 oprofilefs_create_ro_atomic(sb, dir, "bt_lost_no_mapping", 78 oprofilefs_create_ro_atomic(sb, dir, "bt_lost_no_mapping",
78 &oprofile_stats.bt_lost_no_mapping); 79 &oprofile_stats.bt_lost_no_mapping);
80#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
81 oprofilefs_create_ro_atomic(sb, dir, "multiplex_counter",
82 &oprofile_stats.multiplex_counter);
83#endif
79} 84}
diff --git a/drivers/oprofile/oprofile_stats.h b/drivers/oprofile/oprofile_stats.h
index 3da0d08dc1f9..0b54e46c3c14 100644
--- a/drivers/oprofile/oprofile_stats.h
+++ b/drivers/oprofile/oprofile_stats.h
@@ -17,6 +17,7 @@ struct oprofile_stat_struct {
17 atomic_t sample_lost_no_mapping; 17 atomic_t sample_lost_no_mapping;
18 atomic_t bt_lost_no_mapping; 18 atomic_t bt_lost_no_mapping;
19 atomic_t event_lost_overflow; 19 atomic_t event_lost_overflow;
20 atomic_t multiplex_counter;
20}; 21};
21 22
22extern struct oprofile_stat_struct oprofile_stats; 23extern struct oprofile_stat_struct oprofile_stats;
diff --git a/include/linux/oprofile.h b/include/linux/oprofile.h
index 1d9518bc4c58..5171639ecf0f 100644
--- a/include/linux/oprofile.h
+++ b/include/linux/oprofile.h
@@ -67,6 +67,9 @@ struct oprofile_operations {
67 67
68 /* Initiate a stack backtrace. Optional. */ 68 /* Initiate a stack backtrace. Optional. */
69 void (*backtrace)(struct pt_regs * const regs, unsigned int depth); 69 void (*backtrace)(struct pt_regs * const regs, unsigned int depth);
70
71 /* Multiplex between different events. Optional. */
72 int (*switch_events)(void);
70 /* CPU identification string. */ 73 /* CPU identification string. */
71 char * cpu_type; 74 char * cpu_type;
72}; 75};
@@ -171,7 +174,6 @@ struct op_sample;
171struct op_entry { 174struct op_entry {
172 struct ring_buffer_event *event; 175 struct ring_buffer_event *event;
173 struct op_sample *sample; 176 struct op_sample *sample;
174 unsigned long irq_flags;
175 unsigned long size; 177 unsigned long size;
176 unsigned long *data; 178 unsigned long *data;
177}; 179};
@@ -180,6 +182,7 @@ void oprofile_write_reserve(struct op_entry *entry,
180 struct pt_regs * const regs, 182 struct pt_regs * const regs,
181 unsigned long pc, int code, int size); 183 unsigned long pc, int code, int size);
182int oprofile_add_data(struct op_entry *entry, unsigned long val); 184int oprofile_add_data(struct op_entry *entry, unsigned long val);
185int oprofile_add_data64(struct op_entry *entry, u64 val);
183int oprofile_write_commit(struct op_entry *entry); 186int oprofile_write_commit(struct op_entry *entry);
184 187
185#endif /* OPROFILE_H */ 188#endif /* OPROFILE_H */
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-26 07:58:58 -0500