aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2010-10-21 15:54:49 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2010-10-21 15:54:49 -0400
commit5d70f79b5ef6ea2de4f72a37b2d96e2601e40a22 (patch)
treea0d6de0930ba83ecf4629c2e2e261f5eaa2d8f33 /arch/arm/kernel
parent888a6f77e0418b049f83d37547c209b904d30af4 (diff)
parent750ed158bf6c782d2813da1bca2c824365a0b777 (diff)
Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (163 commits) tracing: Fix compile issue for trace_sched_wakeup.c [S390] hardirq: remove pointless header file includes [IA64] Move local_softirq_pending() definition perf, powerpc: Fix power_pmu_event_init to not use event->ctx ftrace: Remove recursion between recordmcount and scripts/mod/empty jump_label: Add COND_STMT(), reducer wrappery perf: Optimize sw events perf: Use jump_labels to optimize the scheduler hooks jump_label: Add atomic_t interface jump_label: Use more consistent naming perf, hw_breakpoint: Fix crash in hw_breakpoint creation perf: Find task before event alloc perf: Fix task refcount bugs perf: Fix group moving irq_work: Add generic hardirq context callbacks perf_events: Fix transaction recovery in group_sched_in() perf_events: Fix bogus AMD64 generic TLB events perf_events: Fix bogus context time tracking tracing: Remove parent recording in latency tracer graph options tracing: Use one prologue for the preempt irqs off tracer function tracers ...
Diffstat (limited to 'arch/arm/kernel')
-rw-r--r--arch/arm/kernel/perf_event.c212
1 files changed, 102 insertions, 110 deletions
diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c
index ecbb0288e5dd..49643b1467e6 100644
--- a/arch/arm/kernel/perf_event.c
+++ b/arch/arm/kernel/perf_event.c
@@ -123,6 +123,12 @@ armpmu_get_max_events(void)
123} 123}
124EXPORT_SYMBOL_GPL(armpmu_get_max_events); 124EXPORT_SYMBOL_GPL(armpmu_get_max_events);
125 125
126int perf_num_counters(void)
127{
128 return armpmu_get_max_events();
129}
130EXPORT_SYMBOL_GPL(perf_num_counters);
131
126#define HW_OP_UNSUPPORTED 0xFFFF 132#define HW_OP_UNSUPPORTED 0xFFFF
127 133
128#define C(_x) \ 134#define C(_x) \
@@ -221,46 +227,56 @@ again:
221} 227}
222 228
223static void 229static void
224armpmu_disable(struct perf_event *event) 230armpmu_read(struct perf_event *event)
225{ 231{
226 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
227 struct hw_perf_event *hwc = &event->hw; 232 struct hw_perf_event *hwc = &event->hw;
228 int idx = hwc->idx;
229
230 WARN_ON(idx < 0);
231
232 clear_bit(idx, cpuc->active_mask);
233 armpmu->disable(hwc, idx);
234
235 barrier();
236 233
237 armpmu_event_update(event, hwc, idx); 234 /* Don't read disabled counters! */
238 cpuc->events[idx] = NULL; 235 if (hwc->idx < 0)
239 clear_bit(idx, cpuc->used_mask); 236 return;
240 237
241 perf_event_update_userpage(event); 238 armpmu_event_update(event, hwc, hwc->idx);
242} 239}
243 240
244static void 241static void
245armpmu_read(struct perf_event *event) 242armpmu_stop(struct perf_event *event, int flags)
246{ 243{
247 struct hw_perf_event *hwc = &event->hw; 244 struct hw_perf_event *hwc = &event->hw;
248 245
249 /* Don't read disabled counters! */ 246 if (!armpmu)
250 if (hwc->idx < 0)
251 return; 247 return;
252 248
253 armpmu_event_update(event, hwc, hwc->idx); 249 /*
250 * ARM pmu always has to update the counter, so ignore
251 * PERF_EF_UPDATE, see comments in armpmu_start().
252 */
253 if (!(hwc->state & PERF_HES_STOPPED)) {
254 armpmu->disable(hwc, hwc->idx);
255 barrier(); /* why? */
256 armpmu_event_update(event, hwc, hwc->idx);
257 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
258 }
254} 259}
255 260
256static void 261static void
257armpmu_unthrottle(struct perf_event *event) 262armpmu_start(struct perf_event *event, int flags)
258{ 263{
259 struct hw_perf_event *hwc = &event->hw; 264 struct hw_perf_event *hwc = &event->hw;
260 265
266 if (!armpmu)
267 return;
268
269 /*
270 * ARM pmu always has to reprogram the period, so ignore
271 * PERF_EF_RELOAD, see the comment below.
272 */
273 if (flags & PERF_EF_RELOAD)
274 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
275
276 hwc->state = 0;
261 /* 277 /*
262 * Set the period again. Some counters can't be stopped, so when we 278 * Set the period again. Some counters can't be stopped, so when we
263 * were throttled we simply disabled the IRQ source and the counter 279 * were stopped we simply disabled the IRQ source and the counter
264 * may have been left counting. If we don't do this step then we may 280 * may have been left counting. If we don't do this step then we may
265 * get an interrupt too soon or *way* too late if the overflow has 281 * get an interrupt too soon or *way* too late if the overflow has
266 * happened since disabling. 282 * happened since disabling.
@@ -269,14 +285,33 @@ armpmu_unthrottle(struct perf_event *event)
269 armpmu->enable(hwc, hwc->idx); 285 armpmu->enable(hwc, hwc->idx);
270} 286}
271 287
288static void
289armpmu_del(struct perf_event *event, int flags)
290{
291 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
292 struct hw_perf_event *hwc = &event->hw;
293 int idx = hwc->idx;
294
295 WARN_ON(idx < 0);
296
297 clear_bit(idx, cpuc->active_mask);
298 armpmu_stop(event, PERF_EF_UPDATE);
299 cpuc->events[idx] = NULL;
300 clear_bit(idx, cpuc->used_mask);
301
302 perf_event_update_userpage(event);
303}
304
272static int 305static int
273armpmu_enable(struct perf_event *event) 306armpmu_add(struct perf_event *event, int flags)
274{ 307{
275 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 308 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
276 struct hw_perf_event *hwc = &event->hw; 309 struct hw_perf_event *hwc = &event->hw;
277 int idx; 310 int idx;
278 int err = 0; 311 int err = 0;
279 312
313 perf_pmu_disable(event->pmu);
314
280 /* If we don't have a space for the counter then finish early. */ 315 /* If we don't have a space for the counter then finish early. */
281 idx = armpmu->get_event_idx(cpuc, hwc); 316 idx = armpmu->get_event_idx(cpuc, hwc);
282 if (idx < 0) { 317 if (idx < 0) {
@@ -293,25 +328,19 @@ armpmu_enable(struct perf_event *event)
293 cpuc->events[idx] = event; 328 cpuc->events[idx] = event;
294 set_bit(idx, cpuc->active_mask); 329 set_bit(idx, cpuc->active_mask);
295 330
296 /* Set the period for the event. */ 331 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
297 armpmu_event_set_period(event, hwc, idx); 332 if (flags & PERF_EF_START)
298 333 armpmu_start(event, PERF_EF_RELOAD);
299 /* Enable the event. */
300 armpmu->enable(hwc, idx);
301 334
302 /* Propagate our changes to the userspace mapping. */ 335 /* Propagate our changes to the userspace mapping. */
303 perf_event_update_userpage(event); 336 perf_event_update_userpage(event);
304 337
305out: 338out:
339 perf_pmu_enable(event->pmu);
306 return err; 340 return err;
307} 341}
308 342
309static struct pmu pmu = { 343static struct pmu pmu;
310 .enable = armpmu_enable,
311 .disable = armpmu_disable,
312 .unthrottle = armpmu_unthrottle,
313 .read = armpmu_read,
314};
315 344
316static int 345static int
317validate_event(struct cpu_hw_events *cpuc, 346validate_event(struct cpu_hw_events *cpuc,
@@ -491,20 +520,29 @@ __hw_perf_event_init(struct perf_event *event)
491 return err; 520 return err;
492} 521}
493 522
494const struct pmu * 523static int armpmu_event_init(struct perf_event *event)
495hw_perf_event_init(struct perf_event *event)
496{ 524{
497 int err = 0; 525 int err = 0;
498 526
527 switch (event->attr.type) {
528 case PERF_TYPE_RAW:
529 case PERF_TYPE_HARDWARE:
530 case PERF_TYPE_HW_CACHE:
531 break;
532
533 default:
534 return -ENOENT;
535 }
536
499 if (!armpmu) 537 if (!armpmu)
500 return ERR_PTR(-ENODEV); 538 return -ENODEV;
501 539
502 event->destroy = hw_perf_event_destroy; 540 event->destroy = hw_perf_event_destroy;
503 541
504 if (!atomic_inc_not_zero(&active_events)) { 542 if (!atomic_inc_not_zero(&active_events)) {
505 if (atomic_read(&active_events) > perf_max_events) { 543 if (atomic_read(&active_events) > armpmu->num_events) {
506 atomic_dec(&active_events); 544 atomic_dec(&active_events);
507 return ERR_PTR(-ENOSPC); 545 return -ENOSPC;
508 } 546 }
509 547
510 mutex_lock(&pmu_reserve_mutex); 548 mutex_lock(&pmu_reserve_mutex);
@@ -518,17 +556,16 @@ hw_perf_event_init(struct perf_event *event)
518 } 556 }
519 557
520 if (err) 558 if (err)
521 return ERR_PTR(err); 559 return err;
522 560
523 err = __hw_perf_event_init(event); 561 err = __hw_perf_event_init(event);
524 if (err) 562 if (err)
525 hw_perf_event_destroy(event); 563 hw_perf_event_destroy(event);
526 564
527 return err ? ERR_PTR(err) : &pmu; 565 return err;
528} 566}
529 567
530void 568static void armpmu_enable(struct pmu *pmu)
531hw_perf_enable(void)
532{ 569{
533 /* Enable all of the perf events on hardware. */ 570 /* Enable all of the perf events on hardware. */
534 int idx; 571 int idx;
@@ -549,13 +586,23 @@ hw_perf_enable(void)
549 armpmu->start(); 586 armpmu->start();
550} 587}
551 588
552void 589static void armpmu_disable(struct pmu *pmu)
553hw_perf_disable(void)
554{ 590{
555 if (armpmu) 591 if (armpmu)
556 armpmu->stop(); 592 armpmu->stop();
557} 593}
558 594
595static struct pmu pmu = {
596 .pmu_enable = armpmu_enable,
597 .pmu_disable = armpmu_disable,
598 .event_init = armpmu_event_init,
599 .add = armpmu_add,
600 .del = armpmu_del,
601 .start = armpmu_start,
602 .stop = armpmu_stop,
603 .read = armpmu_read,
604};
605
559/* 606/*
560 * ARMv6 Performance counter handling code. 607 * ARMv6 Performance counter handling code.
561 * 608 *
@@ -1045,7 +1092,7 @@ armv6pmu_handle_irq(int irq_num,
1045 * platforms that can have the PMU interrupts raised as an NMI, this 1092 * platforms that can have the PMU interrupts raised as an NMI, this
1046 * will not work. 1093 * will not work.
1047 */ 1094 */
1048 perf_event_do_pending(); 1095 irq_work_run();
1049 1096
1050 return IRQ_HANDLED; 1097 return IRQ_HANDLED;
1051} 1098}
@@ -2021,7 +2068,7 @@ static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
2021 * platforms that can have the PMU interrupts raised as an NMI, this 2068 * platforms that can have the PMU interrupts raised as an NMI, this
2022 * will not work. 2069 * will not work.
2023 */ 2070 */
2024 perf_event_do_pending(); 2071 irq_work_run();
2025 2072
2026 return IRQ_HANDLED; 2073 return IRQ_HANDLED;
2027} 2074}
@@ -2389,7 +2436,7 @@ xscale1pmu_handle_irq(int irq_num, void *dev)
2389 armpmu->disable(hwc, idx); 2436 armpmu->disable(hwc, idx);
2390 } 2437 }
2391 2438
2392 perf_event_do_pending(); 2439 irq_work_run();
2393 2440
2394 /* 2441 /*
2395 * Re-enable the PMU. 2442 * Re-enable the PMU.
@@ -2716,7 +2763,7 @@ xscale2pmu_handle_irq(int irq_num, void *dev)
2716 armpmu->disable(hwc, idx); 2763 armpmu->disable(hwc, idx);
2717 } 2764 }
2718 2765
2719 perf_event_do_pending(); 2766 irq_work_run();
2720 2767
2721 /* 2768 /*
2722 * Re-enable the PMU. 2769 * Re-enable the PMU.
@@ -2933,14 +2980,12 @@ init_hw_perf_events(void)
2933 armpmu = &armv6pmu; 2980 armpmu = &armv6pmu;
2934 memcpy(armpmu_perf_cache_map, armv6_perf_cache_map, 2981 memcpy(armpmu_perf_cache_map, armv6_perf_cache_map,
2935 sizeof(armv6_perf_cache_map)); 2982 sizeof(armv6_perf_cache_map));
2936 perf_max_events = armv6pmu.num_events;
2937 break; 2983 break;
2938 case 0xB020: /* ARM11mpcore */ 2984 case 0xB020: /* ARM11mpcore */
2939 armpmu = &armv6mpcore_pmu; 2985 armpmu = &armv6mpcore_pmu;
2940 memcpy(armpmu_perf_cache_map, 2986 memcpy(armpmu_perf_cache_map,
2941 armv6mpcore_perf_cache_map, 2987 armv6mpcore_perf_cache_map,
2942 sizeof(armv6mpcore_perf_cache_map)); 2988 sizeof(armv6mpcore_perf_cache_map));
2943 perf_max_events = armv6mpcore_pmu.num_events;
2944 break; 2989 break;
2945 case 0xC080: /* Cortex-A8 */ 2990 case 0xC080: /* Cortex-A8 */
2946 armv7pmu.id = ARM_PERF_PMU_ID_CA8; 2991 armv7pmu.id = ARM_PERF_PMU_ID_CA8;
@@ -2952,7 +2997,6 @@ init_hw_perf_events(void)
2952 /* Reset PMNC and read the nb of CNTx counters 2997 /* Reset PMNC and read the nb of CNTx counters
2953 supported */ 2998 supported */
2954 armv7pmu.num_events = armv7_reset_read_pmnc(); 2999 armv7pmu.num_events = armv7_reset_read_pmnc();
2955 perf_max_events = armv7pmu.num_events;
2956 break; 3000 break;
2957 case 0xC090: /* Cortex-A9 */ 3001 case 0xC090: /* Cortex-A9 */
2958 armv7pmu.id = ARM_PERF_PMU_ID_CA9; 3002 armv7pmu.id = ARM_PERF_PMU_ID_CA9;
@@ -2964,7 +3008,6 @@ init_hw_perf_events(void)
2964 /* Reset PMNC and read the nb of CNTx counters 3008 /* Reset PMNC and read the nb of CNTx counters
2965 supported */ 3009 supported */
2966 armv7pmu.num_events = armv7_reset_read_pmnc(); 3010 armv7pmu.num_events = armv7_reset_read_pmnc();
2967 perf_max_events = armv7pmu.num_events;
2968 break; 3011 break;
2969 } 3012 }
2970 /* Intel CPUs [xscale]. */ 3013 /* Intel CPUs [xscale]. */
@@ -2975,13 +3018,11 @@ init_hw_perf_events(void)
2975 armpmu = &xscale1pmu; 3018 armpmu = &xscale1pmu;
2976 memcpy(armpmu_perf_cache_map, xscale_perf_cache_map, 3019 memcpy(armpmu_perf_cache_map, xscale_perf_cache_map,
2977 sizeof(xscale_perf_cache_map)); 3020 sizeof(xscale_perf_cache_map));
2978 perf_max_events = xscale1pmu.num_events;
2979 break; 3021 break;
2980 case 2: 3022 case 2:
2981 armpmu = &xscale2pmu; 3023 armpmu = &xscale2pmu;
2982 memcpy(armpmu_perf_cache_map, xscale_perf_cache_map, 3024 memcpy(armpmu_perf_cache_map, xscale_perf_cache_map,
2983 sizeof(xscale_perf_cache_map)); 3025 sizeof(xscale_perf_cache_map));
2984 perf_max_events = xscale2pmu.num_events;
2985 break; 3026 break;
2986 } 3027 }
2987 } 3028 }
@@ -2991,9 +3032,10 @@ init_hw_perf_events(void)
2991 arm_pmu_names[armpmu->id], armpmu->num_events); 3032 arm_pmu_names[armpmu->id], armpmu->num_events);
2992 } else { 3033 } else {
2993 pr_info("no hardware support available\n"); 3034 pr_info("no hardware support available\n");
2994 perf_max_events = -1;
2995 } 3035 }
2996 3036
3037 perf_pmu_register(&pmu);
3038
2997 return 0; 3039 return 0;
2998} 3040}
2999arch_initcall(init_hw_perf_events); 3041arch_initcall(init_hw_perf_events);
@@ -3001,13 +3043,6 @@ arch_initcall(init_hw_perf_events);
3001/* 3043/*
3002 * Callchain handling code. 3044 * Callchain handling code.
3003 */ 3045 */
3004static inline void
3005callchain_store(struct perf_callchain_entry *entry,
3006 u64 ip)
3007{
3008 if (entry->nr < PERF_MAX_STACK_DEPTH)
3009 entry->ip[entry->nr++] = ip;
3010}
3011 3046
3012/* 3047/*
3013 * The registers we're interested in are at the end of the variable 3048 * The registers we're interested in are at the end of the variable
@@ -3039,7 +3074,7 @@ user_backtrace(struct frame_tail *tail,
3039 if (__copy_from_user_inatomic(&buftail, tail, sizeof(buftail))) 3074 if (__copy_from_user_inatomic(&buftail, tail, sizeof(buftail)))
3040 return NULL; 3075 return NULL;
3041 3076
3042 callchain_store(entry, buftail.lr); 3077 perf_callchain_store(entry, buftail.lr);
3043 3078
3044 /* 3079 /*
3045 * Frame pointers should strictly progress back up the stack 3080 * Frame pointers should strictly progress back up the stack
@@ -3051,16 +3086,11 @@ user_backtrace(struct frame_tail *tail,
3051 return buftail.fp - 1; 3086 return buftail.fp - 1;
3052} 3087}
3053 3088
3054static void 3089void
3055perf_callchain_user(struct pt_regs *regs, 3090perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
3056 struct perf_callchain_entry *entry)
3057{ 3091{
3058 struct frame_tail *tail; 3092 struct frame_tail *tail;
3059 3093
3060 callchain_store(entry, PERF_CONTEXT_USER);
3061
3062 if (!user_mode(regs))
3063 regs = task_pt_regs(current);
3064 3094
3065 tail = (struct frame_tail *)regs->ARM_fp - 1; 3095 tail = (struct frame_tail *)regs->ARM_fp - 1;
3066 3096
@@ -3078,56 +3108,18 @@ callchain_trace(struct stackframe *fr,
3078 void *data) 3108 void *data)
3079{ 3109{
3080 struct perf_callchain_entry *entry = data; 3110 struct perf_callchain_entry *entry = data;
3081 callchain_store(entry, fr->pc); 3111 perf_callchain_store(entry, fr->pc);
3082 return 0; 3112 return 0;
3083} 3113}
3084 3114
3085static void 3115void
3086perf_callchain_kernel(struct pt_regs *regs, 3116perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
3087 struct perf_callchain_entry *entry)
3088{ 3117{
3089 struct stackframe fr; 3118 struct stackframe fr;
3090 3119
3091 callchain_store(entry, PERF_CONTEXT_KERNEL);
3092 fr.fp = regs->ARM_fp; 3120 fr.fp = regs->ARM_fp;
3093 fr.sp = regs->ARM_sp; 3121 fr.sp = regs->ARM_sp;
3094 fr.lr = regs->ARM_lr; 3122 fr.lr = regs->ARM_lr;
3095 fr.pc = regs->ARM_pc; 3123 fr.pc = regs->ARM_pc;
3096 walk_stackframe(&fr, callchain_trace, entry); 3124 walk_stackframe(&fr, callchain_trace, entry);
3097} 3125}
3098
3099static void
3100perf_do_callchain(struct pt_regs *regs,
3101 struct perf_callchain_entry *entry)
3102{
3103 int is_user;
3104
3105 if (!regs)
3106 return;
3107
3108 is_user = user_mode(regs);
3109
3110 if (!current || !current->pid)
3111 return;
3112
3113 if (is_user && current->state != TASK_RUNNING)
3114 return;
3115
3116 if (!is_user)
3117 perf_callchain_kernel(regs, entry);
3118
3119 if (current->mm)
3120 perf_callchain_user(regs, entry);
3121}
3122
3123static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
3124
3125struct perf_callchain_entry *
3126perf_callchain(struct pt_regs *regs)
3127{
3128 struct perf_callchain_entry *entry = &__get_cpu_var(pmc_irq_entry);
3129
3130 entry->nr = 0;
3131 perf_do_callchain(regs, entry);
3132 return entry;
3133}