aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/mips/Kconfig2
-rw-r--r--arch/mips/kernel/Makefile5
-rw-r--r--arch/mips/kernel/perf_event.c515
-rw-r--r--arch/mips/kernel/perf_event_mipsxx.c530
4 files changed, 532 insertions, 520 deletions
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 0674b12841ee..90877154f298 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -2101,7 +2101,7 @@ config NODES_SHIFT
2101 2101
2102config HW_PERF_EVENTS 2102config HW_PERF_EVENTS
2103 bool "Enable hardware performance counter support for perf events" 2103 bool "Enable hardware performance counter support for perf events"
2104 depends on PERF_EVENTS && !MIPS_MT_SMTC && OPROFILE=n && CPU_MIPS32 2104 depends on PERF_EVENTS && !MIPS_MT_SMTC && OPROFILE=n && (CPU_MIPS32 || CPU_MIPS64 || CPU_R10000 || CPU_SB1)
2105 default y 2105 default y
2106 help 2106 help
2107 Enable hardware performance counter support for perf events. If 2107 Enable hardware performance counter support for perf events. If
diff --git a/arch/mips/kernel/Makefile b/arch/mips/kernel/Makefile
index 83bba332bbfc..1a966183e353 100644
--- a/arch/mips/kernel/Makefile
+++ b/arch/mips/kernel/Makefile
@@ -11,6 +11,8 @@ obj-y += cpu-probe.o branch.o entry.o genex.o irq.o process.o \
11ifdef CONFIG_FUNCTION_TRACER 11ifdef CONFIG_FUNCTION_TRACER
12CFLAGS_REMOVE_ftrace.o = -pg 12CFLAGS_REMOVE_ftrace.o = -pg
13CFLAGS_REMOVE_early_printk.o = -pg 13CFLAGS_REMOVE_early_printk.o = -pg
14CFLAGS_REMOVE_perf_event.o = -pg
15CFLAGS_REMOVE_perf_event_mipsxx.o = -pg
14endif 16endif
15 17
16obj-$(CONFIG_CEVT_BCM1480) += cevt-bcm1480.o 18obj-$(CONFIG_CEVT_BCM1480) += cevt-bcm1480.o
@@ -106,7 +108,8 @@ obj-$(CONFIG_HAVE_STD_PC_SERIAL_PORT) += 8250-platform.o
106 108
107obj-$(CONFIG_MIPS_CPUFREQ) += cpufreq/ 109obj-$(CONFIG_MIPS_CPUFREQ) += cpufreq/
108 110
109obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o 111obj-$(CONFIG_PERF_EVENTS) += perf_event.o
112obj-$(CONFIG_HW_PERF_EVENTS) += perf_event_mipsxx.o
110 113
111obj-$(CONFIG_JUMP_LABEL) += jump_label.o 114obj-$(CONFIG_JUMP_LABEL) += jump_label.o
112 115
diff --git a/arch/mips/kernel/perf_event.c b/arch/mips/kernel/perf_event.c
index f3d6e99ab575..c1cf9c6c3f77 100644
--- a/arch/mips/kernel/perf_event.c
+++ b/arch/mips/kernel/perf_event.c
@@ -14,529 +14,16 @@
14 * published by the Free Software Foundation. 14 * published by the Free Software Foundation.
15 */ 15 */
16 16
17#include <linux/cpumask.h>
18#include <linux/interrupt.h>
19#include <linux/smp.h>
20#include <linux/kernel.h>
21#include <linux/perf_event.h> 17#include <linux/perf_event.h>
22#include <linux/uaccess.h>
23 18
24#include <asm/irq.h>
25#include <asm/irq_regs.h>
26#include <asm/stacktrace.h> 19#include <asm/stacktrace.h>
27#include <asm/time.h> /* For perf_irq */
28
29/* These are for 32bit counters. For 64bit ones, define them accordingly. */
30#define MAX_PERIOD ((1ULL << 32) - 1)
31#define VALID_COUNT 0x7fffffff
32#define TOTAL_BITS 32
33#define HIGHEST_BIT 31
34
35#define MIPS_MAX_HWEVENTS 4
36
37struct cpu_hw_events {
38 /* Array of events on this cpu. */
39 struct perf_event *events[MIPS_MAX_HWEVENTS];
40
41 /*
42 * Set the bit (indexed by the counter number) when the counter
43 * is used for an event.
44 */
45 unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
46
47 /*
48 * The borrowed MSB for the performance counter. A MIPS performance
49 * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit
50 * counters) as a factor of determining whether a counter overflow
51 * should be signaled. So here we use a separate MSB for each
52 * counter to make things easy.
53 */
54 unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
55
56 /*
57 * Software copy of the control register for each performance counter.
58 * MIPS CPUs vary in performance counters. They use this differently,
59 * and even may not use it.
60 */
61 unsigned int saved_ctrl[MIPS_MAX_HWEVENTS];
62};
63DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
64 .saved_ctrl = {0},
65};
66
67/* The description of MIPS performance events. */
68struct mips_perf_event {
69 unsigned int event_id;
70 /*
71 * MIPS performance counters are indexed starting from 0.
72 * CNTR_EVEN indicates the indexes of the counters to be used are
73 * even numbers.
74 */
75 unsigned int cntr_mask;
76 #define CNTR_EVEN 0x55555555
77 #define CNTR_ODD 0xaaaaaaaa
78#ifdef CONFIG_MIPS_MT_SMP
79 enum {
80 T = 0,
81 V = 1,
82 P = 2,
83 } range;
84#else
85 #define T
86 #define V
87 #define P
88#endif
89};
90
91static struct mips_perf_event raw_event;
92static DEFINE_MUTEX(raw_event_mutex);
93
94#define UNSUPPORTED_PERF_EVENT_ID 0xffffffff
95#define C(x) PERF_COUNT_HW_CACHE_##x
96
97struct mips_pmu {
98 const char *name;
99 int irq;
100 irqreturn_t (*handle_irq)(int irq, void *dev);
101 int (*handle_shared_irq)(void);
102 void (*start)(void);
103 void (*stop)(void);
104 int (*alloc_counter)(struct cpu_hw_events *cpuc,
105 struct hw_perf_event *hwc);
106 u64 (*read_counter)(unsigned int idx);
107 void (*write_counter)(unsigned int idx, u64 val);
108 void (*enable_event)(struct hw_perf_event *evt, int idx);
109 void (*disable_event)(int idx);
110 const struct mips_perf_event *(*map_raw_event)(u64 config);
111 const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
112 const struct mips_perf_event (*cache_event_map)
113 [PERF_COUNT_HW_CACHE_MAX]
114 [PERF_COUNT_HW_CACHE_OP_MAX]
115 [PERF_COUNT_HW_CACHE_RESULT_MAX];
116 unsigned int num_counters;
117};
118
119static const struct mips_pmu *mipspmu;
120
121static int mipspmu_event_set_period(struct perf_event *event,
122 struct hw_perf_event *hwc,
123 int idx)
124{
125 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
126 s64 left = local64_read(&hwc->period_left);
127 s64 period = hwc->sample_period;
128 int ret = 0;
129 u64 uleft;
130 unsigned long flags;
131
132 if (unlikely(left <= -period)) {
133 left = period;
134 local64_set(&hwc->period_left, left);
135 hwc->last_period = period;
136 ret = 1;
137 }
138
139 if (unlikely(left <= 0)) {
140 left += period;
141 local64_set(&hwc->period_left, left);
142 hwc->last_period = period;
143 ret = 1;
144 }
145
146 if (left > (s64)MAX_PERIOD)
147 left = MAX_PERIOD;
148
149 local64_set(&hwc->prev_count, (u64)-left);
150
151 local_irq_save(flags);
152 uleft = (u64)(-left) & MAX_PERIOD;
153 uleft > VALID_COUNT ?
154 set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs);
155 mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT);
156 local_irq_restore(flags);
157
158 perf_event_update_userpage(event);
159
160 return ret;
161}
162
163static void mipspmu_event_update(struct perf_event *event,
164 struct hw_perf_event *hwc,
165 int idx)
166{
167 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
168 unsigned long flags;
169 int shift = 64 - TOTAL_BITS;
170 s64 prev_raw_count, new_raw_count;
171 u64 delta;
172
173again:
174 prev_raw_count = local64_read(&hwc->prev_count);
175 local_irq_save(flags);
176 /* Make the counter value be a "real" one. */
177 new_raw_count = mipspmu->read_counter(idx);
178 if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) {
179 new_raw_count &= VALID_COUNT;
180 clear_bit(idx, cpuc->msbs);
181 } else
182 new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT);
183 local_irq_restore(flags);
184
185 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
186 new_raw_count) != prev_raw_count)
187 goto again;
188
189 delta = (new_raw_count << shift) - (prev_raw_count << shift);
190 delta >>= shift;
191
192 local64_add(delta, &event->count);
193 local64_sub(delta, &hwc->period_left);
194}
195
196static void mipspmu_start(struct perf_event *event, int flags)
197{
198 struct hw_perf_event *hwc = &event->hw;
199
200 if (!mipspmu)
201 return;
202
203 if (flags & PERF_EF_RELOAD)
204 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
205
206 hwc->state = 0;
207
208 /* Set the period for the event. */
209 mipspmu_event_set_period(event, hwc, hwc->idx);
210
211 /* Enable the event. */
212 mipspmu->enable_event(hwc, hwc->idx);
213}
214
215static void mipspmu_stop(struct perf_event *event, int flags)
216{
217 struct hw_perf_event *hwc = &event->hw;
218
219 if (!mipspmu)
220 return;
221
222 if (!(hwc->state & PERF_HES_STOPPED)) {
223 /* We are working on a local event. */
224 mipspmu->disable_event(hwc->idx);
225 barrier();
226 mipspmu_event_update(event, hwc, hwc->idx);
227 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
228 }
229}
230
231static int mipspmu_add(struct perf_event *event, int flags)
232{
233 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
234 struct hw_perf_event *hwc = &event->hw;
235 int idx;
236 int err = 0;
237
238 perf_pmu_disable(event->pmu);
239
240 /* To look for a free counter for this event. */
241 idx = mipspmu->alloc_counter(cpuc, hwc);
242 if (idx < 0) {
243 err = idx;
244 goto out;
245 }
246
247 /*
248 * If there is an event in the counter we are going to use then
249 * make sure it is disabled.
250 */
251 event->hw.idx = idx;
252 mipspmu->disable_event(idx);
253 cpuc->events[idx] = event;
254
255 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
256 if (flags & PERF_EF_START)
257 mipspmu_start(event, PERF_EF_RELOAD);
258
259 /* Propagate our changes to the userspace mapping. */
260 perf_event_update_userpage(event);
261
262out:
263 perf_pmu_enable(event->pmu);
264 return err;
265}
266
267static void mipspmu_del(struct perf_event *event, int flags)
268{
269 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
270 struct hw_perf_event *hwc = &event->hw;
271 int idx = hwc->idx;
272
273 WARN_ON(idx < 0 || idx >= mipspmu->num_counters);
274
275 mipspmu_stop(event, PERF_EF_UPDATE);
276 cpuc->events[idx] = NULL;
277 clear_bit(idx, cpuc->used_mask);
278
279 perf_event_update_userpage(event);
280}
281
282static void mipspmu_read(struct perf_event *event)
283{
284 struct hw_perf_event *hwc = &event->hw;
285
286 /* Don't read disabled counters! */
287 if (hwc->idx < 0)
288 return;
289
290 mipspmu_event_update(event, hwc, hwc->idx);
291}
292
293static void mipspmu_enable(struct pmu *pmu)
294{
295 if (mipspmu)
296 mipspmu->start();
297}
298
299static void mipspmu_disable(struct pmu *pmu)
300{
301 if (mipspmu)
302 mipspmu->stop();
303}
304
305static atomic_t active_events = ATOMIC_INIT(0);
306static DEFINE_MUTEX(pmu_reserve_mutex);
307static int (*save_perf_irq)(void);
308
309static int mipspmu_get_irq(void)
310{
311 int err;
312
313 if (mipspmu->irq >= 0) {
314 /* Request my own irq handler. */
315 err = request_irq(mipspmu->irq, mipspmu->handle_irq,
316 IRQF_DISABLED | IRQF_NOBALANCING,
317 "mips_perf_pmu", NULL);
318 if (err) {
319 pr_warning("Unable to request IRQ%d for MIPS "
320 "performance counters!\n", mipspmu->irq);
321 }
322 } else if (cp0_perfcount_irq < 0) {
323 /*
324 * We are sharing the irq number with the timer interrupt.
325 */
326 save_perf_irq = perf_irq;
327 perf_irq = mipspmu->handle_shared_irq;
328 err = 0;
329 } else {
330 pr_warning("The platform hasn't properly defined its "
331 "interrupt controller.\n");
332 err = -ENOENT;
333 }
334
335 return err;
336}
337
338static void mipspmu_free_irq(void)
339{
340 if (mipspmu->irq >= 0)
341 free_irq(mipspmu->irq, NULL);
342 else if (cp0_perfcount_irq < 0)
343 perf_irq = save_perf_irq;
344}
345
346/*
347 * mipsxx/rm9000/loongson2 have different performance counters, they have
348 * specific low-level init routines.
349 */
350static void reset_counters(void *arg);
351static int __hw_perf_event_init(struct perf_event *event);
352
353static void hw_perf_event_destroy(struct perf_event *event)
354{
355 if (atomic_dec_and_mutex_lock(&active_events,
356 &pmu_reserve_mutex)) {
357 /*
358 * We must not call the destroy function with interrupts
359 * disabled.
360 */
361 on_each_cpu(reset_counters,
362 (void *)(long)mipspmu->num_counters, 1);
363 mipspmu_free_irq();
364 mutex_unlock(&pmu_reserve_mutex);
365 }
366}
367
368static int mipspmu_event_init(struct perf_event *event)
369{
370 int err = 0;
371
372 switch (event->attr.type) {
373 case PERF_TYPE_RAW:
374 case PERF_TYPE_HARDWARE:
375 case PERF_TYPE_HW_CACHE:
376 break;
377
378 default:
379 return -ENOENT;
380 }
381
382 if (!mipspmu || event->cpu >= nr_cpumask_bits ||
383 (event->cpu >= 0 && !cpu_online(event->cpu)))
384 return -ENODEV;
385
386 if (!atomic_inc_not_zero(&active_events)) {
387 if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) {
388 atomic_dec(&active_events);
389 return -ENOSPC;
390 }
391
392 mutex_lock(&pmu_reserve_mutex);
393 if (atomic_read(&active_events) == 0)
394 err = mipspmu_get_irq();
395
396 if (!err)
397 atomic_inc(&active_events);
398 mutex_unlock(&pmu_reserve_mutex);
399 }
400
401 if (err)
402 return err;
403
404 err = __hw_perf_event_init(event);
405 if (err)
406 hw_perf_event_destroy(event);
407
408 return err;
409}
410
411static struct pmu pmu = {
412 .pmu_enable = mipspmu_enable,
413 .pmu_disable = mipspmu_disable,
414 .event_init = mipspmu_event_init,
415 .add = mipspmu_add,
416 .del = mipspmu_del,
417 .start = mipspmu_start,
418 .stop = mipspmu_stop,
419 .read = mipspmu_read,
420};
421
422static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev)
423{
424/*
425 * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for
426 * event_id.
427 */
428#ifdef CONFIG_MIPS_MT_SMP
429 return ((unsigned int)pev->range << 24) |
430 (pev->cntr_mask & 0xffff00) |
431 (pev->event_id & 0xff);
432#else
433 return (pev->cntr_mask & 0xffff00) |
434 (pev->event_id & 0xff);
435#endif
436}
437
438static const struct mips_perf_event *mipspmu_map_general_event(int idx)
439{
440 const struct mips_perf_event *pev;
441
442 pev = ((*mipspmu->general_event_map)[idx].event_id ==
443 UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) :
444 &(*mipspmu->general_event_map)[idx]);
445
446 return pev;
447}
448
449static const struct mips_perf_event *mipspmu_map_cache_event(u64 config)
450{
451 unsigned int cache_type, cache_op, cache_result;
452 const struct mips_perf_event *pev;
453
454 cache_type = (config >> 0) & 0xff;
455 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
456 return ERR_PTR(-EINVAL);
457
458 cache_op = (config >> 8) & 0xff;
459 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
460 return ERR_PTR(-EINVAL);
461
462 cache_result = (config >> 16) & 0xff;
463 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
464 return ERR_PTR(-EINVAL);
465
466 pev = &((*mipspmu->cache_event_map)
467 [cache_type]
468 [cache_op]
469 [cache_result]);
470
471 if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID)
472 return ERR_PTR(-EOPNOTSUPP);
473
474 return pev;
475
476}
477
478static int validate_event(struct cpu_hw_events *cpuc,
479 struct perf_event *event)
480{
481 struct hw_perf_event fake_hwc = event->hw;
482
483 /* Allow mixed event group. So return 1 to pass validation. */
484 if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF)
485 return 1;
486
487 return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0;
488}
489
490static int validate_group(struct perf_event *event)
491{
492 struct perf_event *sibling, *leader = event->group_leader;
493 struct cpu_hw_events fake_cpuc;
494
495 memset(&fake_cpuc, 0, sizeof(fake_cpuc));
496
497 if (!validate_event(&fake_cpuc, leader))
498 return -ENOSPC;
499
500 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
501 if (!validate_event(&fake_cpuc, sibling))
502 return -ENOSPC;
503 }
504
505 if (!validate_event(&fake_cpuc, event))
506 return -ENOSPC;
507
508 return 0;
509}
510
511/* This is needed by specific irq handlers in perf_event_*.c */
512static void handle_associated_event(struct cpu_hw_events *cpuc,
513 int idx, struct perf_sample_data *data,
514 struct pt_regs *regs)
515{
516 struct perf_event *event = cpuc->events[idx];
517 struct hw_perf_event *hwc = &event->hw;
518
519 mipspmu_event_update(event, hwc, idx);
520 data->period = event->hw.last_period;
521 if (!mipspmu_event_set_period(event, hwc, idx))
522 return;
523
524 if (perf_event_overflow(event, data, regs))
525 mipspmu->disable_event(idx);
526}
527
528#include "perf_event_mipsxx.c"
529 20
530/* Callchain handling code. */ 21/* Callchain handling code. */
531 22
532/* 23/*
533 * Leave userspace callchain empty for now. When we find a way to trace 24 * Leave userspace callchain empty for now. When we find a way to trace
534 * the user stack callchains, we add here. 25 * the user stack callchains, we will add it here.
535 */ 26 */
536void perf_callchain_user(struct perf_callchain_entry *entry,
537 struct pt_regs *regs)
538{
539}
540 27
541static void save_raw_perf_callchain(struct perf_callchain_entry *entry, 28static void save_raw_perf_callchain(struct perf_callchain_entry *entry,
542 unsigned long reg29) 29 unsigned long reg29)
diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c
index a5925b5c792b..eb74dce69c1b 100644
--- a/arch/mips/kernel/perf_event_mipsxx.c
+++ b/arch/mips/kernel/perf_event_mipsxx.c
@@ -1,5 +1,529 @@
1#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) || \ 1/*
2 defined(CONFIG_CPU_R10000) || defined(CONFIG_CPU_SB1) 2 * Linux performance counter support for MIPS.
3 *
4 * Copyright (C) 2010 MIPS Technologies, Inc.
5 * Author: Deng-Cheng Zhu
6 *
7 * This code is based on the implementation for ARM, which is in turn
8 * based on the sparc64 perf event code and the x86 code. Performance
9 * counter access is based on the MIPS Oprofile code. And the callchain
10 * support references the code of MIPS stacktrace.c.
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
15 */
16
17#include <linux/cpumask.h>
18#include <linux/interrupt.h>
19#include <linux/smp.h>
20#include <linux/kernel.h>
21#include <linux/perf_event.h>
22#include <linux/uaccess.h>
23
24#include <asm/irq.h>
25#include <asm/irq_regs.h>
26#include <asm/stacktrace.h>
27#include <asm/time.h> /* For perf_irq */
28
29/* These are for 32bit counters. For 64bit ones, define them accordingly. */
30#define MAX_PERIOD ((1ULL << 32) - 1)
31#define VALID_COUNT 0x7fffffff
32#define TOTAL_BITS 32
33#define HIGHEST_BIT 31
34
35#define MIPS_MAX_HWEVENTS 4
36
37struct cpu_hw_events {
38 /* Array of events on this cpu. */
39 struct perf_event *events[MIPS_MAX_HWEVENTS];
40
41 /*
42 * Set the bit (indexed by the counter number) when the counter
43 * is used for an event.
44 */
45 unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
46
47 /*
48 * The borrowed MSB for the performance counter. A MIPS performance
49 * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit
50 * counters) as a factor of determining whether a counter overflow
51 * should be signaled. So here we use a separate MSB for each
52 * counter to make things easy.
53 */
54 unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
55
56 /*
57 * Software copy of the control register for each performance counter.
58 * MIPS CPUs vary in performance counters. They use this differently,
59 * and even may not use it.
60 */
61 unsigned int saved_ctrl[MIPS_MAX_HWEVENTS];
62};
63DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
64 .saved_ctrl = {0},
65};
66
67/* The description of MIPS performance events. */
68struct mips_perf_event {
69 unsigned int event_id;
70 /*
71 * MIPS performance counters are indexed starting from 0.
72 * CNTR_EVEN indicates the indexes of the counters to be used are
73 * even numbers.
74 */
75 unsigned int cntr_mask;
76 #define CNTR_EVEN 0x55555555
77 #define CNTR_ODD 0xaaaaaaaa
78#ifdef CONFIG_MIPS_MT_SMP
79 enum {
80 T = 0,
81 V = 1,
82 P = 2,
83 } range;
84#else
85 #define T
86 #define V
87 #define P
88#endif
89};
90
91static struct mips_perf_event raw_event;
92static DEFINE_MUTEX(raw_event_mutex);
93
94#define UNSUPPORTED_PERF_EVENT_ID 0xffffffff
95#define C(x) PERF_COUNT_HW_CACHE_##x
96
97struct mips_pmu {
98 const char *name;
99 int irq;
100 irqreturn_t (*handle_irq)(int irq, void *dev);
101 int (*handle_shared_irq)(void);
102 void (*start)(void);
103 void (*stop)(void);
104 int (*alloc_counter)(struct cpu_hw_events *cpuc,
105 struct hw_perf_event *hwc);
106 u64 (*read_counter)(unsigned int idx);
107 void (*write_counter)(unsigned int idx, u64 val);
108 void (*enable_event)(struct hw_perf_event *evt, int idx);
109 void (*disable_event)(int idx);
110 const struct mips_perf_event *(*map_raw_event)(u64 config);
111 const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
112 const struct mips_perf_event (*cache_event_map)
113 [PERF_COUNT_HW_CACHE_MAX]
114 [PERF_COUNT_HW_CACHE_OP_MAX]
115 [PERF_COUNT_HW_CACHE_RESULT_MAX];
116 unsigned int num_counters;
117};
118
119static const struct mips_pmu *mipspmu;
120
121static int mipspmu_event_set_period(struct perf_event *event,
122 struct hw_perf_event *hwc,
123 int idx)
124{
125 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
126 s64 left = local64_read(&hwc->period_left);
127 s64 period = hwc->sample_period;
128 int ret = 0;
129 u64 uleft;
130 unsigned long flags;
131
132 if (unlikely(left <= -period)) {
133 left = period;
134 local64_set(&hwc->period_left, left);
135 hwc->last_period = period;
136 ret = 1;
137 }
138
139 if (unlikely(left <= 0)) {
140 left += period;
141 local64_set(&hwc->period_left, left);
142 hwc->last_period = period;
143 ret = 1;
144 }
145
146 if (left > (s64)MAX_PERIOD)
147 left = MAX_PERIOD;
148
149 local64_set(&hwc->prev_count, (u64)-left);
150
151 local_irq_save(flags);
152 uleft = (u64)(-left) & MAX_PERIOD;
153 uleft > VALID_COUNT ?
154 set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs);
155 mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT);
156 local_irq_restore(flags);
157
158 perf_event_update_userpage(event);
159
160 return ret;
161}
162
163static void mipspmu_event_update(struct perf_event *event,
164 struct hw_perf_event *hwc,
165 int idx)
166{
167 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
168 unsigned long flags;
169 int shift = 64 - TOTAL_BITS;
170 s64 prev_raw_count, new_raw_count;
171 u64 delta;
172
173again:
174 prev_raw_count = local64_read(&hwc->prev_count);
175 local_irq_save(flags);
176 /* Make the counter value be a "real" one. */
177 new_raw_count = mipspmu->read_counter(idx);
178 if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) {
179 new_raw_count &= VALID_COUNT;
180 clear_bit(idx, cpuc->msbs);
181 } else
182 new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT);
183 local_irq_restore(flags);
184
185 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
186 new_raw_count) != prev_raw_count)
187 goto again;
188
189 delta = (new_raw_count << shift) - (prev_raw_count << shift);
190 delta >>= shift;
191
192 local64_add(delta, &event->count);
193 local64_sub(delta, &hwc->period_left);
194}
195
196static void mipspmu_start(struct perf_event *event, int flags)
197{
198 struct hw_perf_event *hwc = &event->hw;
199
200 if (!mipspmu)
201 return;
202
203 if (flags & PERF_EF_RELOAD)
204 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
205
206 hwc->state = 0;
207
208 /* Set the period for the event. */
209 mipspmu_event_set_period(event, hwc, hwc->idx);
210
211 /* Enable the event. */
212 mipspmu->enable_event(hwc, hwc->idx);
213}
214
215static void mipspmu_stop(struct perf_event *event, int flags)
216{
217 struct hw_perf_event *hwc = &event->hw;
218
219 if (!mipspmu)
220 return;
221
222 if (!(hwc->state & PERF_HES_STOPPED)) {
223 /* We are working on a local event. */
224 mipspmu->disable_event(hwc->idx);
225 barrier();
226 mipspmu_event_update(event, hwc, hwc->idx);
227 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
228 }
229}
230
231static int mipspmu_add(struct perf_event *event, int flags)
232{
233 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
234 struct hw_perf_event *hwc = &event->hw;
235 int idx;
236 int err = 0;
237
238 perf_pmu_disable(event->pmu);
239
240 /* To look for a free counter for this event. */
241 idx = mipspmu->alloc_counter(cpuc, hwc);
242 if (idx < 0) {
243 err = idx;
244 goto out;
245 }
246
247 /*
248 * If there is an event in the counter we are going to use then
249 * make sure it is disabled.
250 */
251 event->hw.idx = idx;
252 mipspmu->disable_event(idx);
253 cpuc->events[idx] = event;
254
255 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
256 if (flags & PERF_EF_START)
257 mipspmu_start(event, PERF_EF_RELOAD);
258
259 /* Propagate our changes to the userspace mapping. */
260 perf_event_update_userpage(event);
261
262out:
263 perf_pmu_enable(event->pmu);
264 return err;
265}
266
267static void mipspmu_del(struct perf_event *event, int flags)
268{
269 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
270 struct hw_perf_event *hwc = &event->hw;
271 int idx = hwc->idx;
272
273 WARN_ON(idx < 0 || idx >= mipspmu->num_counters);
274
275 mipspmu_stop(event, PERF_EF_UPDATE);
276 cpuc->events[idx] = NULL;
277 clear_bit(idx, cpuc->used_mask);
278
279 perf_event_update_userpage(event);
280}
281
282static void mipspmu_read(struct perf_event *event)
283{
284 struct hw_perf_event *hwc = &event->hw;
285
286 /* Don't read disabled counters! */
287 if (hwc->idx < 0)
288 return;
289
290 mipspmu_event_update(event, hwc, hwc->idx);
291}
292
293static void mipspmu_enable(struct pmu *pmu)
294{
295 if (mipspmu)
296 mipspmu->start();
297}
298
299static void mipspmu_disable(struct pmu *pmu)
300{
301 if (mipspmu)
302 mipspmu->stop();
303}
304
305static atomic_t active_events = ATOMIC_INIT(0);
306static DEFINE_MUTEX(pmu_reserve_mutex);
307static int (*save_perf_irq)(void);
308
309static int mipspmu_get_irq(void)
310{
311 int err;
312
313 if (mipspmu->irq >= 0) {
314 /* Request my own irq handler. */
315 err = request_irq(mipspmu->irq, mipspmu->handle_irq,
316 IRQF_DISABLED | IRQF_NOBALANCING,
317 "mips_perf_pmu", NULL);
318 if (err) {
319 pr_warning("Unable to request IRQ%d for MIPS "
320 "performance counters!\n", mipspmu->irq);
321 }
322 } else if (cp0_perfcount_irq < 0) {
323 /*
324 * We are sharing the irq number with the timer interrupt.
325 */
326 save_perf_irq = perf_irq;
327 perf_irq = mipspmu->handle_shared_irq;
328 err = 0;
329 } else {
330 pr_warning("The platform hasn't properly defined its "
331 "interrupt controller.\n");
332 err = -ENOENT;
333 }
334
335 return err;
336}
337
338static void mipspmu_free_irq(void)
339{
340 if (mipspmu->irq >= 0)
341 free_irq(mipspmu->irq, NULL);
342 else if (cp0_perfcount_irq < 0)
343 perf_irq = save_perf_irq;
344}
345
346/*
347 * mipsxx/rm9000/loongson2 have different performance counters, they have
348 * specific low-level init routines.
349 */
350static void reset_counters(void *arg);
351static int __hw_perf_event_init(struct perf_event *event);
352
353static void hw_perf_event_destroy(struct perf_event *event)
354{
355 if (atomic_dec_and_mutex_lock(&active_events,
356 &pmu_reserve_mutex)) {
357 /*
358 * We must not call the destroy function with interrupts
359 * disabled.
360 */
361 on_each_cpu(reset_counters,
362 (void *)(long)mipspmu->num_counters, 1);
363 mipspmu_free_irq();
364 mutex_unlock(&pmu_reserve_mutex);
365 }
366}
367
368static int mipspmu_event_init(struct perf_event *event)
369{
370 int err = 0;
371
372 switch (event->attr.type) {
373 case PERF_TYPE_RAW:
374 case PERF_TYPE_HARDWARE:
375 case PERF_TYPE_HW_CACHE:
376 break;
377
378 default:
379 return -ENOENT;
380 }
381
382 if (!mipspmu || event->cpu >= nr_cpumask_bits ||
383 (event->cpu >= 0 && !cpu_online(event->cpu)))
384 return -ENODEV;
385
386 if (!atomic_inc_not_zero(&active_events)) {
387 if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) {
388 atomic_dec(&active_events);
389 return -ENOSPC;
390 }
391
392 mutex_lock(&pmu_reserve_mutex);
393 if (atomic_read(&active_events) == 0)
394 err = mipspmu_get_irq();
395
396 if (!err)
397 atomic_inc(&active_events);
398 mutex_unlock(&pmu_reserve_mutex);
399 }
400
401 if (err)
402 return err;
403
404 err = __hw_perf_event_init(event);
405 if (err)
406 hw_perf_event_destroy(event);
407
408 return err;
409}
410
411static struct pmu pmu = {
412 .pmu_enable = mipspmu_enable,
413 .pmu_disable = mipspmu_disable,
414 .event_init = mipspmu_event_init,
415 .add = mipspmu_add,
416 .del = mipspmu_del,
417 .start = mipspmu_start,
418 .stop = mipspmu_stop,
419 .read = mipspmu_read,
420};
421
422static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev)
423{
424/*
425 * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for
426 * event_id.
427 */
428#ifdef CONFIG_MIPS_MT_SMP
429 return ((unsigned int)pev->range << 24) |
430 (pev->cntr_mask & 0xffff00) |
431 (pev->event_id & 0xff);
432#else
433 return (pev->cntr_mask & 0xffff00) |
434 (pev->event_id & 0xff);
435#endif
436}
437
438static const struct mips_perf_event *mipspmu_map_general_event(int idx)
439{
440 const struct mips_perf_event *pev;
441
442 pev = ((*mipspmu->general_event_map)[idx].event_id ==
443 UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) :
444 &(*mipspmu->general_event_map)[idx]);
445
446 return pev;
447}
448
449static const struct mips_perf_event *mipspmu_map_cache_event(u64 config)
450{
451 unsigned int cache_type, cache_op, cache_result;
452 const struct mips_perf_event *pev;
453
454 cache_type = (config >> 0) & 0xff;
455 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
456 return ERR_PTR(-EINVAL);
457
458 cache_op = (config >> 8) & 0xff;
459 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
460 return ERR_PTR(-EINVAL);
461
462 cache_result = (config >> 16) & 0xff;
463 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
464 return ERR_PTR(-EINVAL);
465
466 pev = &((*mipspmu->cache_event_map)
467 [cache_type]
468 [cache_op]
469 [cache_result]);
470
471 if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID)
472 return ERR_PTR(-EOPNOTSUPP);
473
474 return pev;
475
476}
477
478static int validate_event(struct cpu_hw_events *cpuc,
479 struct perf_event *event)
480{
481 struct hw_perf_event fake_hwc = event->hw;
482
483 /* Allow mixed event group. So return 1 to pass validation. */
484 if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF)
485 return 1;
486
487 return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0;
488}
489
490static int validate_group(struct perf_event *event)
491{
492 struct perf_event *sibling, *leader = event->group_leader;
493 struct cpu_hw_events fake_cpuc;
494
495 memset(&fake_cpuc, 0, sizeof(fake_cpuc));
496
497 if (!validate_event(&fake_cpuc, leader))
498 return -ENOSPC;
499
500 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
501 if (!validate_event(&fake_cpuc, sibling))
502 return -ENOSPC;
503 }
504
505 if (!validate_event(&fake_cpuc, event))
506 return -ENOSPC;
507
508 return 0;
509}
510
511/* This is needed by specific irq handlers in perf_event_*.c */
512static void handle_associated_event(struct cpu_hw_events *cpuc,
513 int idx, struct perf_sample_data *data,
514 struct pt_regs *regs)
515{
516 struct perf_event *event = cpuc->events[idx];
517 struct hw_perf_event *hwc = &event->hw;
518
519 mipspmu_event_update(event, hwc, idx);
520 data->period = event->hw.last_period;
521 if (!mipspmu_event_set_period(event, hwc, idx))
522 return;
523
524 if (perf_event_overflow(event, data, regs))
525 mipspmu->disable_event(idx);
526}
3 527
4#define M_CONFIG1_PC (1 << 4) 528#define M_CONFIG1_PC (1 << 4)
5 529
@@ -1062,5 +1586,3 @@ init_hw_perf_events(void)
1062 return 0; 1586 return 0;
1063} 1587}
1064early_initcall(init_hw_perf_events); 1588early_initcall(init_hw_perf_events);
1065
1066#endif /* defined(CONFIG_CPU_MIPS32)... */
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-31 14:32:09 -0500