diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2011-12-05 19:54:00 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2011-12-05 19:54:00 -0500 |
commit | 232ea344550c4a099d35d9df552509d6748a31c0 (patch) | |
tree | 2b4a1c351d290537fdf18cc5bed78bf43536f175 | |
parent | 40c043b077c6e377c8440d71563c055d0c4f0f0a (diff) | |
parent | dc440d10e1668b15fe704a23adb9b53fbbb24a44 (diff) |
Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
* 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
perf: Fix loss of notification with multi-event
perf, x86: Force IBS LVT offset assignment for family 10h
perf, x86: Disable PEBS on SandyBridge chips
trace_events_filter: Use rcu_assign_pointer() when setting ftrace_event_call->filter
perf session: Fix crash with invalid CPU list
perf python: Fix undefined symbol problem
perf/x86: Enable raw event access to Intel offcore events
perf: Don't use -ENOSPC for out of PMU resources
perf: Do not set task_ctx pointer in cpuctx if there are no events in the context
perf/x86: Fix PEBS instruction unwind
oprofile, x86: Fix crash when unloading module (nmi timer mode)
oprofile: Fix crash when unloading module (hr timer mode)
-rw-r--r-- | arch/arm/kernel/perf_event.c | 6 | ||||
-rw-r--r-- | arch/mips/kernel/perf_event_mipsxx.c | 8 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/perf_event.c | 16 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/perf_event_amd_ibs.c | 29 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/perf_event_intel.c | 8 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/perf_event_intel_ds.c | 6 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/perf_event_p4.c | 2 | ||||
-rw-r--r-- | arch/x86/oprofile/init.c | 7 | ||||
-rw-r--r-- | drivers/oprofile/oprof.c | 29 | ||||
-rw-r--r-- | drivers/oprofile/timer_int.c | 1 | ||||
-rw-r--r-- | include/linux/perf_event.h | 1 | ||||
-rw-r--r-- | kernel/events/core.c | 89 | ||||
-rw-r--r-- | kernel/events/internal.h | 3 | ||||
-rw-r--r-- | kernel/events/ring_buffer.c | 3 | ||||
-rw-r--r-- | kernel/trace/trace_events_filter.c | 6 | ||||
-rw-r--r-- | tools/perf/util/evsel.c | 10 | ||||
-rw-r--r-- | tools/perf/util/hist.c | 10 | ||||
-rw-r--r-- | tools/perf/util/hist.h | 2 | ||||
-rw-r--r-- | tools/perf/util/session.c | 4 |
19 files changed, 185 insertions, 55 deletions
diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c index c475379199b1..8e9c98edc068 100644 --- a/arch/arm/kernel/perf_event.c +++ b/arch/arm/kernel/perf_event.c | |||
@@ -353,15 +353,15 @@ validate_group(struct perf_event *event) | |||
353 | fake_pmu.used_mask = fake_used_mask; | 353 | fake_pmu.used_mask = fake_used_mask; |
354 | 354 | ||
355 | if (!validate_event(&fake_pmu, leader)) | 355 | if (!validate_event(&fake_pmu, leader)) |
356 | return -ENOSPC; | 356 | return -EINVAL; |
357 | 357 | ||
358 | list_for_each_entry(sibling, &leader->sibling_list, group_entry) { | 358 | list_for_each_entry(sibling, &leader->sibling_list, group_entry) { |
359 | if (!validate_event(&fake_pmu, sibling)) | 359 | if (!validate_event(&fake_pmu, sibling)) |
360 | return -ENOSPC; | 360 | return -EINVAL; |
361 | } | 361 | } |
362 | 362 | ||
363 | if (!validate_event(&fake_pmu, event)) | 363 | if (!validate_event(&fake_pmu, event)) |
364 | return -ENOSPC; | 364 | return -EINVAL; |
365 | 365 | ||
366 | return 0; | 366 | return 0; |
367 | } | 367 | } |
diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index 4f2971bcf8e5..315fc0b250f8 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c | |||
@@ -623,7 +623,7 @@ static int mipspmu_event_init(struct perf_event *event) | |||
623 | if (!atomic_inc_not_zero(&active_events)) { | 623 | if (!atomic_inc_not_zero(&active_events)) { |
624 | if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) { | 624 | if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) { |
625 | atomic_dec(&active_events); | 625 | atomic_dec(&active_events); |
626 | return -ENOSPC; | 626 | return -EINVAL; |
627 | } | 627 | } |
628 | 628 | ||
629 | mutex_lock(&pmu_reserve_mutex); | 629 | mutex_lock(&pmu_reserve_mutex); |
@@ -732,15 +732,15 @@ static int validate_group(struct perf_event *event) | |||
732 | memset(&fake_cpuc, 0, sizeof(fake_cpuc)); | 732 | memset(&fake_cpuc, 0, sizeof(fake_cpuc)); |
733 | 733 | ||
734 | if (!validate_event(&fake_cpuc, leader)) | 734 | if (!validate_event(&fake_cpuc, leader)) |
735 | return -ENOSPC; | 735 | return -EINVAL; |
736 | 736 | ||
737 | list_for_each_entry(sibling, &leader->sibling_list, group_entry) { | 737 | list_for_each_entry(sibling, &leader->sibling_list, group_entry) { |
738 | if (!validate_event(&fake_cpuc, sibling)) | 738 | if (!validate_event(&fake_cpuc, sibling)) |
739 | return -ENOSPC; | 739 | return -EINVAL; |
740 | } | 740 | } |
741 | 741 | ||
742 | if (!validate_event(&fake_cpuc, event)) | 742 | if (!validate_event(&fake_cpuc, event)) |
743 | return -ENOSPC; | 743 | return -EINVAL; |
744 | 744 | ||
745 | return 0; | 745 | return 0; |
746 | } | 746 | } |
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 640891014b2a..2bda212a0010 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c | |||
@@ -312,12 +312,8 @@ int x86_setup_perfctr(struct perf_event *event) | |||
312 | return -EOPNOTSUPP; | 312 | return -EOPNOTSUPP; |
313 | } | 313 | } |
314 | 314 | ||
315 | /* | ||
316 | * Do not allow config1 (extended registers) to propagate, | ||
317 | * there's no sane user-space generalization yet: | ||
318 | */ | ||
319 | if (attr->type == PERF_TYPE_RAW) | 315 | if (attr->type == PERF_TYPE_RAW) |
320 | return 0; | 316 | return x86_pmu_extra_regs(event->attr.config, event); |
321 | 317 | ||
322 | if (attr->type == PERF_TYPE_HW_CACHE) | 318 | if (attr->type == PERF_TYPE_HW_CACHE) |
323 | return set_ext_hw_attr(hwc, event); | 319 | return set_ext_hw_attr(hwc, event); |
@@ -588,7 +584,7 @@ done: | |||
588 | x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]); | 584 | x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]); |
589 | } | 585 | } |
590 | } | 586 | } |
591 | return num ? -ENOSPC : 0; | 587 | return num ? -EINVAL : 0; |
592 | } | 588 | } |
593 | 589 | ||
594 | /* | 590 | /* |
@@ -607,7 +603,7 @@ static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, | |||
607 | 603 | ||
608 | if (is_x86_event(leader)) { | 604 | if (is_x86_event(leader)) { |
609 | if (n >= max_count) | 605 | if (n >= max_count) |
610 | return -ENOSPC; | 606 | return -EINVAL; |
611 | cpuc->event_list[n] = leader; | 607 | cpuc->event_list[n] = leader; |
612 | n++; | 608 | n++; |
613 | } | 609 | } |
@@ -620,7 +616,7 @@ static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, | |||
620 | continue; | 616 | continue; |
621 | 617 | ||
622 | if (n >= max_count) | 618 | if (n >= max_count) |
623 | return -ENOSPC; | 619 | return -EINVAL; |
624 | 620 | ||
625 | cpuc->event_list[n] = event; | 621 | cpuc->event_list[n] = event; |
626 | n++; | 622 | n++; |
@@ -1316,7 +1312,7 @@ static int validate_event(struct perf_event *event) | |||
1316 | c = x86_pmu.get_event_constraints(fake_cpuc, event); | 1312 | c = x86_pmu.get_event_constraints(fake_cpuc, event); |
1317 | 1313 | ||
1318 | if (!c || !c->weight) | 1314 | if (!c || !c->weight) |
1319 | ret = -ENOSPC; | 1315 | ret = -EINVAL; |
1320 | 1316 | ||
1321 | if (x86_pmu.put_event_constraints) | 1317 | if (x86_pmu.put_event_constraints) |
1322 | x86_pmu.put_event_constraints(fake_cpuc, event); | 1318 | x86_pmu.put_event_constraints(fake_cpuc, event); |
@@ -1341,7 +1337,7 @@ static int validate_group(struct perf_event *event) | |||
1341 | { | 1337 | { |
1342 | struct perf_event *leader = event->group_leader; | 1338 | struct perf_event *leader = event->group_leader; |
1343 | struct cpu_hw_events *fake_cpuc; | 1339 | struct cpu_hw_events *fake_cpuc; |
1344 | int ret = -ENOSPC, n; | 1340 | int ret = -EINVAL, n; |
1345 | 1341 | ||
1346 | fake_cpuc = allocate_fake_cpuc(); | 1342 | fake_cpuc = allocate_fake_cpuc(); |
1347 | if (IS_ERR(fake_cpuc)) | 1343 | if (IS_ERR(fake_cpuc)) |
diff --git a/arch/x86/kernel/cpu/perf_event_amd_ibs.c b/arch/x86/kernel/cpu/perf_event_amd_ibs.c index ab6343d21825..3b8a2d30d14e 100644 --- a/arch/x86/kernel/cpu/perf_event_amd_ibs.c +++ b/arch/x86/kernel/cpu/perf_event_amd_ibs.c | |||
@@ -199,8 +199,7 @@ static int force_ibs_eilvt_setup(void) | |||
199 | goto out; | 199 | goto out; |
200 | } | 200 | } |
201 | 201 | ||
202 | pr_err(FW_BUG "using offset %d for IBS interrupts\n", offset); | 202 | pr_info("IBS: LVT offset %d assigned\n", offset); |
203 | pr_err(FW_BUG "workaround enabled for IBS LVT offset\n"); | ||
204 | 203 | ||
205 | return 0; | 204 | return 0; |
206 | out: | 205 | out: |
@@ -265,19 +264,23 @@ perf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *h | |||
265 | static __init int amd_ibs_init(void) | 264 | static __init int amd_ibs_init(void) |
266 | { | 265 | { |
267 | u32 caps; | 266 | u32 caps; |
268 | int ret; | 267 | int ret = -EINVAL; |
269 | 268 | ||
270 | caps = __get_ibs_caps(); | 269 | caps = __get_ibs_caps(); |
271 | if (!caps) | 270 | if (!caps) |
272 | return -ENODEV; /* ibs not supported by the cpu */ | 271 | return -ENODEV; /* ibs not supported by the cpu */ |
273 | 272 | ||
274 | if (!ibs_eilvt_valid()) { | 273 | /* |
275 | ret = force_ibs_eilvt_setup(); | 274 | * Force LVT offset assignment for family 10h: The offsets are |
276 | if (ret) { | 275 | * not assigned by the BIOS for this family, so the OS is |
277 | pr_err("Failed to setup IBS, %d\n", ret); | 276 | * responsible for doing it. If the OS assignment fails, fall |
278 | return ret; | 277 | * back to BIOS settings and try to setup this. |
279 | } | 278 | */ |
280 | } | 279 | if (boot_cpu_data.x86 == 0x10) |
280 | force_ibs_eilvt_setup(); | ||
281 | |||
282 | if (!ibs_eilvt_valid()) | ||
283 | goto out; | ||
281 | 284 | ||
282 | get_online_cpus(); | 285 | get_online_cpus(); |
283 | ibs_caps = caps; | 286 | ibs_caps = caps; |
@@ -287,7 +290,11 @@ static __init int amd_ibs_init(void) | |||
287 | smp_call_function(setup_APIC_ibs, NULL, 1); | 290 | smp_call_function(setup_APIC_ibs, NULL, 1); |
288 | put_online_cpus(); | 291 | put_online_cpus(); |
289 | 292 | ||
290 | return perf_event_ibs_init(); | 293 | ret = perf_event_ibs_init(); |
294 | out: | ||
295 | if (ret) | ||
296 | pr_err("Failed to setup IBS, %d\n", ret); | ||
297 | return ret; | ||
291 | } | 298 | } |
292 | 299 | ||
293 | /* Since we need the pci subsystem to init ibs we can't do this earlier: */ | 300 | /* Since we need the pci subsystem to init ibs we can't do this earlier: */ |
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index 2be5ebe99872..8d601b18bf9f 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c | |||
@@ -1545,6 +1545,13 @@ static void intel_clovertown_quirks(void) | |||
1545 | x86_pmu.pebs_constraints = NULL; | 1545 | x86_pmu.pebs_constraints = NULL; |
1546 | } | 1546 | } |
1547 | 1547 | ||
1548 | static void intel_sandybridge_quirks(void) | ||
1549 | { | ||
1550 | printk(KERN_WARNING "PEBS disabled due to CPU errata.\n"); | ||
1551 | x86_pmu.pebs = 0; | ||
1552 | x86_pmu.pebs_constraints = NULL; | ||
1553 | } | ||
1554 | |||
1548 | __init int intel_pmu_init(void) | 1555 | __init int intel_pmu_init(void) |
1549 | { | 1556 | { |
1550 | union cpuid10_edx edx; | 1557 | union cpuid10_edx edx; |
@@ -1694,6 +1701,7 @@ __init int intel_pmu_init(void) | |||
1694 | break; | 1701 | break; |
1695 | 1702 | ||
1696 | case 42: /* SandyBridge */ | 1703 | case 42: /* SandyBridge */ |
1704 | x86_pmu.quirks = intel_sandybridge_quirks; | ||
1697 | case 45: /* SandyBridge, "Romely-EP" */ | 1705 | case 45: /* SandyBridge, "Romely-EP" */ |
1698 | memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, | 1706 | memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, |
1699 | sizeof(hw_cache_event_ids)); | 1707 | sizeof(hw_cache_event_ids)); |
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c index c0d238f49db8..73da6b64f5b7 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_ds.c +++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c | |||
@@ -493,6 +493,7 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs) | |||
493 | unsigned long from = cpuc->lbr_entries[0].from; | 493 | unsigned long from = cpuc->lbr_entries[0].from; |
494 | unsigned long old_to, to = cpuc->lbr_entries[0].to; | 494 | unsigned long old_to, to = cpuc->lbr_entries[0].to; |
495 | unsigned long ip = regs->ip; | 495 | unsigned long ip = regs->ip; |
496 | int is_64bit = 0; | ||
496 | 497 | ||
497 | /* | 498 | /* |
498 | * We don't need to fixup if the PEBS assist is fault like | 499 | * We don't need to fixup if the PEBS assist is fault like |
@@ -544,7 +545,10 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs) | |||
544 | } else | 545 | } else |
545 | kaddr = (void *)to; | 546 | kaddr = (void *)to; |
546 | 547 | ||
547 | kernel_insn_init(&insn, kaddr); | 548 | #ifdef CONFIG_X86_64 |
549 | is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32); | ||
550 | #endif | ||
551 | insn_init(&insn, kaddr, is_64bit); | ||
548 | insn_get_length(&insn); | 552 | insn_get_length(&insn); |
549 | to += insn.length; | 553 | to += insn.length; |
550 | } while (to < ip); | 554 | } while (to < ip); |
diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c index 492bf1358a7c..ef484d9d0a25 100644 --- a/arch/x86/kernel/cpu/perf_event_p4.c +++ b/arch/x86/kernel/cpu/perf_event_p4.c | |||
@@ -1268,7 +1268,7 @@ reserve: | |||
1268 | } | 1268 | } |
1269 | 1269 | ||
1270 | done: | 1270 | done: |
1271 | return num ? -ENOSPC : 0; | 1271 | return num ? -EINVAL : 0; |
1272 | } | 1272 | } |
1273 | 1273 | ||
1274 | static __initconst const struct x86_pmu p4_pmu = { | 1274 | static __initconst const struct x86_pmu p4_pmu = { |
diff --git a/arch/x86/oprofile/init.c b/arch/x86/oprofile/init.c index cdfe4c54deca..f148cf652678 100644 --- a/arch/x86/oprofile/init.c +++ b/arch/x86/oprofile/init.c | |||
@@ -21,6 +21,7 @@ extern int op_nmi_timer_init(struct oprofile_operations *ops); | |||
21 | extern void op_nmi_exit(void); | 21 | extern void op_nmi_exit(void); |
22 | extern void x86_backtrace(struct pt_regs * const regs, unsigned int depth); | 22 | extern void x86_backtrace(struct pt_regs * const regs, unsigned int depth); |
23 | 23 | ||
24 | static int nmi_timer; | ||
24 | 25 | ||
25 | int __init oprofile_arch_init(struct oprofile_operations *ops) | 26 | int __init oprofile_arch_init(struct oprofile_operations *ops) |
26 | { | 27 | { |
@@ -31,8 +32,9 @@ int __init oprofile_arch_init(struct oprofile_operations *ops) | |||
31 | #ifdef CONFIG_X86_LOCAL_APIC | 32 | #ifdef CONFIG_X86_LOCAL_APIC |
32 | ret = op_nmi_init(ops); | 33 | ret = op_nmi_init(ops); |
33 | #endif | 34 | #endif |
35 | nmi_timer = (ret != 0); | ||
34 | #ifdef CONFIG_X86_IO_APIC | 36 | #ifdef CONFIG_X86_IO_APIC |
35 | if (ret < 0) | 37 | if (nmi_timer) |
36 | ret = op_nmi_timer_init(ops); | 38 | ret = op_nmi_timer_init(ops); |
37 | #endif | 39 | #endif |
38 | ops->backtrace = x86_backtrace; | 40 | ops->backtrace = x86_backtrace; |
@@ -44,6 +46,7 @@ int __init oprofile_arch_init(struct oprofile_operations *ops) | |||
44 | void oprofile_arch_exit(void) | 46 | void oprofile_arch_exit(void) |
45 | { | 47 | { |
46 | #ifdef CONFIG_X86_LOCAL_APIC | 48 | #ifdef CONFIG_X86_LOCAL_APIC |
47 | op_nmi_exit(); | 49 | if (!nmi_timer) |
50 | op_nmi_exit(); | ||
48 | #endif | 51 | #endif |
49 | } | 52 | } |
diff --git a/drivers/oprofile/oprof.c b/drivers/oprofile/oprof.c index dccd8636095c..f8c752e408a6 100644 --- a/drivers/oprofile/oprof.c +++ b/drivers/oprofile/oprof.c | |||
@@ -239,26 +239,45 @@ int oprofile_set_ulong(unsigned long *addr, unsigned long val) | |||
239 | return err; | 239 | return err; |
240 | } | 240 | } |
241 | 241 | ||
242 | static int timer_mode; | ||
243 | |||
242 | static int __init oprofile_init(void) | 244 | static int __init oprofile_init(void) |
243 | { | 245 | { |
244 | int err; | 246 | int err; |
245 | 247 | ||
248 | /* always init architecture to setup backtrace support */ | ||
246 | err = oprofile_arch_init(&oprofile_ops); | 249 | err = oprofile_arch_init(&oprofile_ops); |
247 | if (err < 0 || timer) { | 250 | |
248 | printk(KERN_INFO "oprofile: using timer interrupt.\n"); | 251 | timer_mode = err || timer; /* fall back to timer mode on errors */ |
252 | if (timer_mode) { | ||
253 | if (!err) | ||
254 | oprofile_arch_exit(); | ||
249 | err = oprofile_timer_init(&oprofile_ops); | 255 | err = oprofile_timer_init(&oprofile_ops); |
250 | if (err) | 256 | if (err) |
251 | return err; | 257 | return err; |
252 | } | 258 | } |
253 | return oprofilefs_register(); | 259 | |
260 | err = oprofilefs_register(); | ||
261 | if (!err) | ||
262 | return 0; | ||
263 | |||
264 | /* failed */ | ||
265 | if (timer_mode) | ||
266 | oprofile_timer_exit(); | ||
267 | else | ||
268 | oprofile_arch_exit(); | ||
269 | |||
270 | return err; | ||
254 | } | 271 | } |
255 | 272 | ||
256 | 273 | ||
257 | static void __exit oprofile_exit(void) | 274 | static void __exit oprofile_exit(void) |
258 | { | 275 | { |
259 | oprofile_timer_exit(); | ||
260 | oprofilefs_unregister(); | 276 | oprofilefs_unregister(); |
261 | oprofile_arch_exit(); | 277 | if (timer_mode) |
278 | oprofile_timer_exit(); | ||
279 | else | ||
280 | oprofile_arch_exit(); | ||
262 | } | 281 | } |
263 | 282 | ||
264 | 283 | ||
diff --git a/drivers/oprofile/timer_int.c b/drivers/oprofile/timer_int.c index 3ef44624f510..878fba126582 100644 --- a/drivers/oprofile/timer_int.c +++ b/drivers/oprofile/timer_int.c | |||
@@ -110,6 +110,7 @@ int oprofile_timer_init(struct oprofile_operations *ops) | |||
110 | ops->start = oprofile_hrtimer_start; | 110 | ops->start = oprofile_hrtimer_start; |
111 | ops->stop = oprofile_hrtimer_stop; | 111 | ops->stop = oprofile_hrtimer_stop; |
112 | ops->cpu_type = "timer"; | 112 | ops->cpu_type = "timer"; |
113 | printk(KERN_INFO "oprofile: using timer interrupt.\n"); | ||
113 | return 0; | 114 | return 0; |
114 | } | 115 | } |
115 | 116 | ||
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 1e9ebe5e0091..b1f89122bf6a 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h | |||
@@ -822,6 +822,7 @@ struct perf_event { | |||
822 | int mmap_locked; | 822 | int mmap_locked; |
823 | struct user_struct *mmap_user; | 823 | struct user_struct *mmap_user; |
824 | struct ring_buffer *rb; | 824 | struct ring_buffer *rb; |
825 | struct list_head rb_entry; | ||
825 | 826 | ||
826 | /* poll related */ | 827 | /* poll related */ |
827 | wait_queue_head_t waitq; | 828 | wait_queue_head_t waitq; |
diff --git a/kernel/events/core.c b/kernel/events/core.c index 0e8457da6f95..600c1629b64d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c | |||
@@ -185,6 +185,9 @@ static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, | |||
185 | static void update_context_time(struct perf_event_context *ctx); | 185 | static void update_context_time(struct perf_event_context *ctx); |
186 | static u64 perf_event_time(struct perf_event *event); | 186 | static u64 perf_event_time(struct perf_event *event); |
187 | 187 | ||
188 | static void ring_buffer_attach(struct perf_event *event, | ||
189 | struct ring_buffer *rb); | ||
190 | |||
188 | void __weak perf_event_print_debug(void) { } | 191 | void __weak perf_event_print_debug(void) { } |
189 | 192 | ||
190 | extern __weak const char *perf_pmu_name(void) | 193 | extern __weak const char *perf_pmu_name(void) |
@@ -2173,7 +2176,8 @@ static void perf_event_context_sched_in(struct perf_event_context *ctx, | |||
2173 | 2176 | ||
2174 | perf_event_sched_in(cpuctx, ctx, task); | 2177 | perf_event_sched_in(cpuctx, ctx, task); |
2175 | 2178 | ||
2176 | cpuctx->task_ctx = ctx; | 2179 | if (ctx->nr_events) |
2180 | cpuctx->task_ctx = ctx; | ||
2177 | 2181 | ||
2178 | perf_pmu_enable(ctx->pmu); | 2182 | perf_pmu_enable(ctx->pmu); |
2179 | perf_ctx_unlock(cpuctx, ctx); | 2183 | perf_ctx_unlock(cpuctx, ctx); |
@@ -3190,12 +3194,33 @@ static unsigned int perf_poll(struct file *file, poll_table *wait) | |||
3190 | struct ring_buffer *rb; | 3194 | struct ring_buffer *rb; |
3191 | unsigned int events = POLL_HUP; | 3195 | unsigned int events = POLL_HUP; |
3192 | 3196 | ||
3197 | /* | ||
3198 | * Race between perf_event_set_output() and perf_poll(): perf_poll() | ||
3199 | * grabs the rb reference but perf_event_set_output() overrides it. | ||
3200 | * Here is the timeline for two threads T1, T2: | ||
3201 | * t0: T1, rb = rcu_dereference(event->rb) | ||
3202 | * t1: T2, old_rb = event->rb | ||
3203 | * t2: T2, event->rb = new rb | ||
3204 | * t3: T2, ring_buffer_detach(old_rb) | ||
3205 | * t4: T1, ring_buffer_attach(rb1) | ||
3206 | * t5: T1, poll_wait(event->waitq) | ||
3207 | * | ||
3208 | * To avoid this problem, we grab mmap_mutex in perf_poll() | ||
3209 | * thereby ensuring that the assignment of the new ring buffer | ||
3210 | * and the detachment of the old buffer appear atomic to perf_poll() | ||
3211 | */ | ||
3212 | mutex_lock(&event->mmap_mutex); | ||
3213 | |||
3193 | rcu_read_lock(); | 3214 | rcu_read_lock(); |
3194 | rb = rcu_dereference(event->rb); | 3215 | rb = rcu_dereference(event->rb); |
3195 | if (rb) | 3216 | if (rb) { |
3217 | ring_buffer_attach(event, rb); | ||
3196 | events = atomic_xchg(&rb->poll, 0); | 3218 | events = atomic_xchg(&rb->poll, 0); |
3219 | } | ||
3197 | rcu_read_unlock(); | 3220 | rcu_read_unlock(); |
3198 | 3221 | ||
3222 | mutex_unlock(&event->mmap_mutex); | ||
3223 | |||
3199 | poll_wait(file, &event->waitq, wait); | 3224 | poll_wait(file, &event->waitq, wait); |
3200 | 3225 | ||
3201 | return events; | 3226 | return events; |
@@ -3496,6 +3521,49 @@ unlock: | |||
3496 | return ret; | 3521 | return ret; |
3497 | } | 3522 | } |
3498 | 3523 | ||
3524 | static void ring_buffer_attach(struct perf_event *event, | ||
3525 | struct ring_buffer *rb) | ||
3526 | { | ||
3527 | unsigned long flags; | ||
3528 | |||
3529 | if (!list_empty(&event->rb_entry)) | ||
3530 | return; | ||
3531 | |||
3532 | spin_lock_irqsave(&rb->event_lock, flags); | ||
3533 | if (!list_empty(&event->rb_entry)) | ||
3534 | goto unlock; | ||
3535 | |||
3536 | list_add(&event->rb_entry, &rb->event_list); | ||
3537 | unlock: | ||
3538 | spin_unlock_irqrestore(&rb->event_lock, flags); | ||
3539 | } | ||
3540 | |||
3541 | static void ring_buffer_detach(struct perf_event *event, | ||
3542 | struct ring_buffer *rb) | ||
3543 | { | ||
3544 | unsigned long flags; | ||
3545 | |||
3546 | if (list_empty(&event->rb_entry)) | ||
3547 | return; | ||
3548 | |||
3549 | spin_lock_irqsave(&rb->event_lock, flags); | ||
3550 | list_del_init(&event->rb_entry); | ||
3551 | wake_up_all(&event->waitq); | ||
3552 | spin_unlock_irqrestore(&rb->event_lock, flags); | ||
3553 | } | ||
3554 | |||
3555 | static void ring_buffer_wakeup(struct perf_event *event) | ||
3556 | { | ||
3557 | struct ring_buffer *rb; | ||
3558 | |||
3559 | rcu_read_lock(); | ||
3560 | rb = rcu_dereference(event->rb); | ||
3561 | list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { | ||
3562 | wake_up_all(&event->waitq); | ||
3563 | } | ||
3564 | rcu_read_unlock(); | ||
3565 | } | ||
3566 | |||
3499 | static void rb_free_rcu(struct rcu_head *rcu_head) | 3567 | static void rb_free_rcu(struct rcu_head *rcu_head) |
3500 | { | 3568 | { |
3501 | struct ring_buffer *rb; | 3569 | struct ring_buffer *rb; |
@@ -3521,9 +3589,19 @@ static struct ring_buffer *ring_buffer_get(struct perf_event *event) | |||
3521 | 3589 | ||
3522 | static void ring_buffer_put(struct ring_buffer *rb) | 3590 | static void ring_buffer_put(struct ring_buffer *rb) |
3523 | { | 3591 | { |
3592 | struct perf_event *event, *n; | ||
3593 | unsigned long flags; | ||
3594 | |||
3524 | if (!atomic_dec_and_test(&rb->refcount)) | 3595 | if (!atomic_dec_and_test(&rb->refcount)) |
3525 | return; | 3596 | return; |
3526 | 3597 | ||
3598 | spin_lock_irqsave(&rb->event_lock, flags); | ||
3599 | list_for_each_entry_safe(event, n, &rb->event_list, rb_entry) { | ||
3600 | list_del_init(&event->rb_entry); | ||
3601 | wake_up_all(&event->waitq); | ||
3602 | } | ||
3603 | spin_unlock_irqrestore(&rb->event_lock, flags); | ||
3604 | |||
3527 | call_rcu(&rb->rcu_head, rb_free_rcu); | 3605 | call_rcu(&rb->rcu_head, rb_free_rcu); |
3528 | } | 3606 | } |
3529 | 3607 | ||
@@ -3546,6 +3624,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) | |||
3546 | atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm); | 3624 | atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm); |
3547 | vma->vm_mm->pinned_vm -= event->mmap_locked; | 3625 | vma->vm_mm->pinned_vm -= event->mmap_locked; |
3548 | rcu_assign_pointer(event->rb, NULL); | 3626 | rcu_assign_pointer(event->rb, NULL); |
3627 | ring_buffer_detach(event, rb); | ||
3549 | mutex_unlock(&event->mmap_mutex); | 3628 | mutex_unlock(&event->mmap_mutex); |
3550 | 3629 | ||
3551 | ring_buffer_put(rb); | 3630 | ring_buffer_put(rb); |
@@ -3700,7 +3779,7 @@ static const struct file_operations perf_fops = { | |||
3700 | 3779 | ||
3701 | void perf_event_wakeup(struct perf_event *event) | 3780 | void perf_event_wakeup(struct perf_event *event) |
3702 | { | 3781 | { |
3703 | wake_up_all(&event->waitq); | 3782 | ring_buffer_wakeup(event); |
3704 | 3783 | ||
3705 | if (event->pending_kill) { | 3784 | if (event->pending_kill) { |
3706 | kill_fasync(&event->fasync, SIGIO, event->pending_kill); | 3785 | kill_fasync(&event->fasync, SIGIO, event->pending_kill); |
@@ -5822,6 +5901,8 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, | |||
5822 | INIT_LIST_HEAD(&event->group_entry); | 5901 | INIT_LIST_HEAD(&event->group_entry); |
5823 | INIT_LIST_HEAD(&event->event_entry); | 5902 | INIT_LIST_HEAD(&event->event_entry); |
5824 | INIT_LIST_HEAD(&event->sibling_list); | 5903 | INIT_LIST_HEAD(&event->sibling_list); |
5904 | INIT_LIST_HEAD(&event->rb_entry); | ||
5905 | |||
5825 | init_waitqueue_head(&event->waitq); | 5906 | init_waitqueue_head(&event->waitq); |
5826 | init_irq_work(&event->pending, perf_pending_event); | 5907 | init_irq_work(&event->pending, perf_pending_event); |
5827 | 5908 | ||
@@ -6028,6 +6109,8 @@ set: | |||
6028 | 6109 | ||
6029 | old_rb = event->rb; | 6110 | old_rb = event->rb; |
6030 | rcu_assign_pointer(event->rb, rb); | 6111 | rcu_assign_pointer(event->rb, rb); |
6112 | if (old_rb) | ||
6113 | ring_buffer_detach(event, old_rb); | ||
6031 | ret = 0; | 6114 | ret = 0; |
6032 | unlock: | 6115 | unlock: |
6033 | mutex_unlock(&event->mmap_mutex); | 6116 | mutex_unlock(&event->mmap_mutex); |
diff --git a/kernel/events/internal.h b/kernel/events/internal.h index 09097dd8116c..64568a699375 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h | |||
@@ -22,6 +22,9 @@ struct ring_buffer { | |||
22 | local_t lost; /* nr records lost */ | 22 | local_t lost; /* nr records lost */ |
23 | 23 | ||
24 | long watermark; /* wakeup watermark */ | 24 | long watermark; /* wakeup watermark */ |
25 | /* poll crap */ | ||
26 | spinlock_t event_lock; | ||
27 | struct list_head event_list; | ||
25 | 28 | ||
26 | struct perf_event_mmap_page *user_page; | 29 | struct perf_event_mmap_page *user_page; |
27 | void *data_pages[0]; | 30 | void *data_pages[0]; |
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index a2a29205cc0f..7f3011c6b57f 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c | |||
@@ -209,6 +209,9 @@ ring_buffer_init(struct ring_buffer *rb, long watermark, int flags) | |||
209 | rb->writable = 1; | 209 | rb->writable = 1; |
210 | 210 | ||
211 | atomic_set(&rb->refcount, 1); | 211 | atomic_set(&rb->refcount, 1); |
212 | |||
213 | INIT_LIST_HEAD(&rb->event_list); | ||
214 | spin_lock_init(&rb->event_lock); | ||
212 | } | 215 | } |
213 | 216 | ||
214 | #ifndef CONFIG_PERF_USE_VMALLOC | 217 | #ifndef CONFIG_PERF_USE_VMALLOC |
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 816d3d074979..d6e7926dcd26 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c | |||
@@ -1686,7 +1686,7 @@ static int replace_system_preds(struct event_subsystem *system, | |||
1686 | * replace the filter for the call. | 1686 | * replace the filter for the call. |
1687 | */ | 1687 | */ |
1688 | filter = call->filter; | 1688 | filter = call->filter; |
1689 | call->filter = filter_item->filter; | 1689 | rcu_assign_pointer(call->filter, filter_item->filter); |
1690 | filter_item->filter = filter; | 1690 | filter_item->filter = filter; |
1691 | 1691 | ||
1692 | fail = false; | 1692 | fail = false; |
@@ -1741,7 +1741,7 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string) | |||
1741 | filter = call->filter; | 1741 | filter = call->filter; |
1742 | if (!filter) | 1742 | if (!filter) |
1743 | goto out_unlock; | 1743 | goto out_unlock; |
1744 | call->filter = NULL; | 1744 | RCU_INIT_POINTER(call->filter, NULL); |
1745 | /* Make sure the filter is not being used */ | 1745 | /* Make sure the filter is not being used */ |
1746 | synchronize_sched(); | 1746 | synchronize_sched(); |
1747 | __free_filter(filter); | 1747 | __free_filter(filter); |
@@ -1782,7 +1782,7 @@ out: | |||
1782 | * string | 1782 | * string |
1783 | */ | 1783 | */ |
1784 | tmp = call->filter; | 1784 | tmp = call->filter; |
1785 | call->filter = filter; | 1785 | rcu_assign_pointer(call->filter, filter); |
1786 | if (tmp) { | 1786 | if (tmp) { |
1787 | /* Make sure the call is done with the filter */ | 1787 | /* Make sure the call is done with the filter */ |
1788 | synchronize_sched(); | 1788 | synchronize_sched(); |
diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index e42626422587..d7915d4e77cb 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c | |||
@@ -34,6 +34,16 @@ int __perf_evsel__sample_size(u64 sample_type) | |||
34 | return size; | 34 | return size; |
35 | } | 35 | } |
36 | 36 | ||
37 | static void hists__init(struct hists *hists) | ||
38 | { | ||
39 | memset(hists, 0, sizeof(*hists)); | ||
40 | hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT; | ||
41 | hists->entries_in = &hists->entries_in_array[0]; | ||
42 | hists->entries_collapsed = RB_ROOT; | ||
43 | hists->entries = RB_ROOT; | ||
44 | pthread_mutex_init(&hists->lock, NULL); | ||
45 | } | ||
46 | |||
37 | void perf_evsel__init(struct perf_evsel *evsel, | 47 | void perf_evsel__init(struct perf_evsel *evsel, |
38 | struct perf_event_attr *attr, int idx) | 48 | struct perf_event_attr *attr, int idx) |
39 | { | 49 | { |
diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index a36a3fa81ffb..abef2703cd24 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c | |||
@@ -1211,13 +1211,3 @@ size_t hists__fprintf_nr_events(struct hists *hists, FILE *fp) | |||
1211 | 1211 | ||
1212 | return ret; | 1212 | return ret; |
1213 | } | 1213 | } |
1214 | |||
1215 | void hists__init(struct hists *hists) | ||
1216 | { | ||
1217 | memset(hists, 0, sizeof(*hists)); | ||
1218 | hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT; | ||
1219 | hists->entries_in = &hists->entries_in_array[0]; | ||
1220 | hists->entries_collapsed = RB_ROOT; | ||
1221 | hists->entries = RB_ROOT; | ||
1222 | pthread_mutex_init(&hists->lock, NULL); | ||
1223 | } | ||
diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h index c86c1d27bd1e..89289c8e935e 100644 --- a/tools/perf/util/hist.h +++ b/tools/perf/util/hist.h | |||
@@ -63,8 +63,6 @@ struct hists { | |||
63 | struct callchain_cursor callchain_cursor; | 63 | struct callchain_cursor callchain_cursor; |
64 | }; | 64 | }; |
65 | 65 | ||
66 | void hists__init(struct hists *hists); | ||
67 | |||
68 | struct hist_entry *__hists__add_entry(struct hists *self, | 66 | struct hist_entry *__hists__add_entry(struct hists *self, |
69 | struct addr_location *al, | 67 | struct addr_location *al, |
70 | struct symbol *parent, u64 period); | 68 | struct symbol *parent, u64 period); |
diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index 85c1e6b76f0a..0f4555ce9063 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c | |||
@@ -1333,6 +1333,10 @@ int perf_session__cpu_bitmap(struct perf_session *session, | |||
1333 | } | 1333 | } |
1334 | 1334 | ||
1335 | map = cpu_map__new(cpu_list); | 1335 | map = cpu_map__new(cpu_list); |
1336 | if (map == NULL) { | ||
1337 | pr_err("Invalid cpu_list\n"); | ||
1338 | return -1; | ||
1339 | } | ||
1336 | 1340 | ||
1337 | for (i = 0; i < map->nr; i++) { | 1341 | for (i = 0; i < map->nr; i++) { |
1338 | int cpu = map->map[i]; | 1342 | int cpu = map->map[i]; |