aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2011-12-05 19:54:00 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2011-12-05 19:54:00 -0500
commit232ea344550c4a099d35d9df552509d6748a31c0 (patch)
tree2b4a1c351d290537fdf18cc5bed78bf43536f175
parent40c043b077c6e377c8440d71563c055d0c4f0f0a (diff)
parentdc440d10e1668b15fe704a23adb9b53fbbb24a44 (diff)
Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
* 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: perf: Fix loss of notification with multi-event perf, x86: Force IBS LVT offset assignment for family 10h perf, x86: Disable PEBS on SandyBridge chips trace_events_filter: Use rcu_assign_pointer() when setting ftrace_event_call->filter perf session: Fix crash with invalid CPU list perf python: Fix undefined symbol problem perf/x86: Enable raw event access to Intel offcore events perf: Don't use -ENOSPC for out of PMU resources perf: Do not set task_ctx pointer in cpuctx if there are no events in the context perf/x86: Fix PEBS instruction unwind oprofile, x86: Fix crash when unloading module (nmi timer mode) oprofile: Fix crash when unloading module (hr timer mode)
-rw-r--r--arch/arm/kernel/perf_event.c6
-rw-r--r--arch/mips/kernel/perf_event_mipsxx.c8
-rw-r--r--arch/x86/kernel/cpu/perf_event.c16
-rw-r--r--arch/x86/kernel/cpu/perf_event_amd_ibs.c29
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel.c8
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_ds.c6
-rw-r--r--arch/x86/kernel/cpu/perf_event_p4.c2
-rw-r--r--arch/x86/oprofile/init.c7
-rw-r--r--drivers/oprofile/oprof.c29
-rw-r--r--drivers/oprofile/timer_int.c1
-rw-r--r--include/linux/perf_event.h1
-rw-r--r--kernel/events/core.c89
-rw-r--r--kernel/events/internal.h3
-rw-r--r--kernel/events/ring_buffer.c3
-rw-r--r--kernel/trace/trace_events_filter.c6
-rw-r--r--tools/perf/util/evsel.c10
-rw-r--r--tools/perf/util/hist.c10
-rw-r--r--tools/perf/util/hist.h2
-rw-r--r--tools/perf/util/session.c4
19 files changed, 185 insertions, 55 deletions
diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c
index c475379199b1..8e9c98edc068 100644
--- a/arch/arm/kernel/perf_event.c
+++ b/arch/arm/kernel/perf_event.c
@@ -353,15 +353,15 @@ validate_group(struct perf_event *event)
353 fake_pmu.used_mask = fake_used_mask; 353 fake_pmu.used_mask = fake_used_mask;
354 354
355 if (!validate_event(&fake_pmu, leader)) 355 if (!validate_event(&fake_pmu, leader))
356 return -ENOSPC; 356 return -EINVAL;
357 357
358 list_for_each_entry(sibling, &leader->sibling_list, group_entry) { 358 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
359 if (!validate_event(&fake_pmu, sibling)) 359 if (!validate_event(&fake_pmu, sibling))
360 return -ENOSPC; 360 return -EINVAL;
361 } 361 }
362 362
363 if (!validate_event(&fake_pmu, event)) 363 if (!validate_event(&fake_pmu, event))
364 return -ENOSPC; 364 return -EINVAL;
365 365
366 return 0; 366 return 0;
367} 367}
diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c
index 4f2971bcf8e5..315fc0b250f8 100644
--- a/arch/mips/kernel/perf_event_mipsxx.c
+++ b/arch/mips/kernel/perf_event_mipsxx.c
@@ -623,7 +623,7 @@ static int mipspmu_event_init(struct perf_event *event)
623 if (!atomic_inc_not_zero(&active_events)) { 623 if (!atomic_inc_not_zero(&active_events)) {
624 if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) { 624 if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) {
625 atomic_dec(&active_events); 625 atomic_dec(&active_events);
626 return -ENOSPC; 626 return -EINVAL;
627 } 627 }
628 628
629 mutex_lock(&pmu_reserve_mutex); 629 mutex_lock(&pmu_reserve_mutex);
@@ -732,15 +732,15 @@ static int validate_group(struct perf_event *event)
732 memset(&fake_cpuc, 0, sizeof(fake_cpuc)); 732 memset(&fake_cpuc, 0, sizeof(fake_cpuc));
733 733
734 if (!validate_event(&fake_cpuc, leader)) 734 if (!validate_event(&fake_cpuc, leader))
735 return -ENOSPC; 735 return -EINVAL;
736 736
737 list_for_each_entry(sibling, &leader->sibling_list, group_entry) { 737 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
738 if (!validate_event(&fake_cpuc, sibling)) 738 if (!validate_event(&fake_cpuc, sibling))
739 return -ENOSPC; 739 return -EINVAL;
740 } 740 }
741 741
742 if (!validate_event(&fake_cpuc, event)) 742 if (!validate_event(&fake_cpuc, event))
743 return -ENOSPC; 743 return -EINVAL;
744 744
745 return 0; 745 return 0;
746} 746}
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 640891014b2a..2bda212a0010 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -312,12 +312,8 @@ int x86_setup_perfctr(struct perf_event *event)
312 return -EOPNOTSUPP; 312 return -EOPNOTSUPP;
313 } 313 }
314 314
315 /*
316 * Do not allow config1 (extended registers) to propagate,
317 * there's no sane user-space generalization yet:
318 */
319 if (attr->type == PERF_TYPE_RAW) 315 if (attr->type == PERF_TYPE_RAW)
320 return 0; 316 return x86_pmu_extra_regs(event->attr.config, event);
321 317
322 if (attr->type == PERF_TYPE_HW_CACHE) 318 if (attr->type == PERF_TYPE_HW_CACHE)
323 return set_ext_hw_attr(hwc, event); 319 return set_ext_hw_attr(hwc, event);
@@ -588,7 +584,7 @@ done:
588 x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]); 584 x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
589 } 585 }
590 } 586 }
591 return num ? -ENOSPC : 0; 587 return num ? -EINVAL : 0;
592} 588}
593 589
594/* 590/*
@@ -607,7 +603,7 @@ static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader,
607 603
608 if (is_x86_event(leader)) { 604 if (is_x86_event(leader)) {
609 if (n >= max_count) 605 if (n >= max_count)
610 return -ENOSPC; 606 return -EINVAL;
611 cpuc->event_list[n] = leader; 607 cpuc->event_list[n] = leader;
612 n++; 608 n++;
613 } 609 }
@@ -620,7 +616,7 @@ static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader,
620 continue; 616 continue;
621 617
622 if (n >= max_count) 618 if (n >= max_count)
623 return -ENOSPC; 619 return -EINVAL;
624 620
625 cpuc->event_list[n] = event; 621 cpuc->event_list[n] = event;
626 n++; 622 n++;
@@ -1316,7 +1312,7 @@ static int validate_event(struct perf_event *event)
1316 c = x86_pmu.get_event_constraints(fake_cpuc, event); 1312 c = x86_pmu.get_event_constraints(fake_cpuc, event);
1317 1313
1318 if (!c || !c->weight) 1314 if (!c || !c->weight)
1319 ret = -ENOSPC; 1315 ret = -EINVAL;
1320 1316
1321 if (x86_pmu.put_event_constraints) 1317 if (x86_pmu.put_event_constraints)
1322 x86_pmu.put_event_constraints(fake_cpuc, event); 1318 x86_pmu.put_event_constraints(fake_cpuc, event);
@@ -1341,7 +1337,7 @@ static int validate_group(struct perf_event *event)
1341{ 1337{
1342 struct perf_event *leader = event->group_leader; 1338 struct perf_event *leader = event->group_leader;
1343 struct cpu_hw_events *fake_cpuc; 1339 struct cpu_hw_events *fake_cpuc;
1344 int ret = -ENOSPC, n; 1340 int ret = -EINVAL, n;
1345 1341
1346 fake_cpuc = allocate_fake_cpuc(); 1342 fake_cpuc = allocate_fake_cpuc();
1347 if (IS_ERR(fake_cpuc)) 1343 if (IS_ERR(fake_cpuc))
diff --git a/arch/x86/kernel/cpu/perf_event_amd_ibs.c b/arch/x86/kernel/cpu/perf_event_amd_ibs.c
index ab6343d21825..3b8a2d30d14e 100644
--- a/arch/x86/kernel/cpu/perf_event_amd_ibs.c
+++ b/arch/x86/kernel/cpu/perf_event_amd_ibs.c
@@ -199,8 +199,7 @@ static int force_ibs_eilvt_setup(void)
199 goto out; 199 goto out;
200 } 200 }
201 201
202 pr_err(FW_BUG "using offset %d for IBS interrupts\n", offset); 202 pr_info("IBS: LVT offset %d assigned\n", offset);
203 pr_err(FW_BUG "workaround enabled for IBS LVT offset\n");
204 203
205 return 0; 204 return 0;
206out: 205out:
@@ -265,19 +264,23 @@ perf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *h
265static __init int amd_ibs_init(void) 264static __init int amd_ibs_init(void)
266{ 265{
267 u32 caps; 266 u32 caps;
268 int ret; 267 int ret = -EINVAL;
269 268
270 caps = __get_ibs_caps(); 269 caps = __get_ibs_caps();
271 if (!caps) 270 if (!caps)
272 return -ENODEV; /* ibs not supported by the cpu */ 271 return -ENODEV; /* ibs not supported by the cpu */
273 272
274 if (!ibs_eilvt_valid()) { 273 /*
275 ret = force_ibs_eilvt_setup(); 274 * Force LVT offset assignment for family 10h: The offsets are
276 if (ret) { 275 * not assigned by the BIOS for this family, so the OS is
277 pr_err("Failed to setup IBS, %d\n", ret); 276 * responsible for doing it. If the OS assignment fails, fall
278 return ret; 277 * back to BIOS settings and try to setup this.
279 } 278 */
280 } 279 if (boot_cpu_data.x86 == 0x10)
280 force_ibs_eilvt_setup();
281
282 if (!ibs_eilvt_valid())
283 goto out;
281 284
282 get_online_cpus(); 285 get_online_cpus();
283 ibs_caps = caps; 286 ibs_caps = caps;
@@ -287,7 +290,11 @@ static __init int amd_ibs_init(void)
287 smp_call_function(setup_APIC_ibs, NULL, 1); 290 smp_call_function(setup_APIC_ibs, NULL, 1);
288 put_online_cpus(); 291 put_online_cpus();
289 292
290 return perf_event_ibs_init(); 293 ret = perf_event_ibs_init();
294out:
295 if (ret)
296 pr_err("Failed to setup IBS, %d\n", ret);
297 return ret;
291} 298}
292 299
293/* Since we need the pci subsystem to init ibs we can't do this earlier: */ 300/* Since we need the pci subsystem to init ibs we can't do this earlier: */
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 2be5ebe99872..8d601b18bf9f 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -1545,6 +1545,13 @@ static void intel_clovertown_quirks(void)
1545 x86_pmu.pebs_constraints = NULL; 1545 x86_pmu.pebs_constraints = NULL;
1546} 1546}
1547 1547
1548static void intel_sandybridge_quirks(void)
1549{
1550 printk(KERN_WARNING "PEBS disabled due to CPU errata.\n");
1551 x86_pmu.pebs = 0;
1552 x86_pmu.pebs_constraints = NULL;
1553}
1554
1548__init int intel_pmu_init(void) 1555__init int intel_pmu_init(void)
1549{ 1556{
1550 union cpuid10_edx edx; 1557 union cpuid10_edx edx;
@@ -1694,6 +1701,7 @@ __init int intel_pmu_init(void)
1694 break; 1701 break;
1695 1702
1696 case 42: /* SandyBridge */ 1703 case 42: /* SandyBridge */
1704 x86_pmu.quirks = intel_sandybridge_quirks;
1697 case 45: /* SandyBridge, "Romely-EP" */ 1705 case 45: /* SandyBridge, "Romely-EP" */
1698 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, 1706 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
1699 sizeof(hw_cache_event_ids)); 1707 sizeof(hw_cache_event_ids));
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index c0d238f49db8..73da6b64f5b7 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -493,6 +493,7 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
493 unsigned long from = cpuc->lbr_entries[0].from; 493 unsigned long from = cpuc->lbr_entries[0].from;
494 unsigned long old_to, to = cpuc->lbr_entries[0].to; 494 unsigned long old_to, to = cpuc->lbr_entries[0].to;
495 unsigned long ip = regs->ip; 495 unsigned long ip = regs->ip;
496 int is_64bit = 0;
496 497
497 /* 498 /*
498 * We don't need to fixup if the PEBS assist is fault like 499 * We don't need to fixup if the PEBS assist is fault like
@@ -544,7 +545,10 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
544 } else 545 } else
545 kaddr = (void *)to; 546 kaddr = (void *)to;
546 547
547 kernel_insn_init(&insn, kaddr); 548#ifdef CONFIG_X86_64
549 is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32);
550#endif
551 insn_init(&insn, kaddr, is_64bit);
548 insn_get_length(&insn); 552 insn_get_length(&insn);
549 to += insn.length; 553 to += insn.length;
550 } while (to < ip); 554 } while (to < ip);
diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c
index 492bf1358a7c..ef484d9d0a25 100644
--- a/arch/x86/kernel/cpu/perf_event_p4.c
+++ b/arch/x86/kernel/cpu/perf_event_p4.c
@@ -1268,7 +1268,7 @@ reserve:
1268 } 1268 }
1269 1269
1270done: 1270done:
1271 return num ? -ENOSPC : 0; 1271 return num ? -EINVAL : 0;
1272} 1272}
1273 1273
1274static __initconst const struct x86_pmu p4_pmu = { 1274static __initconst const struct x86_pmu p4_pmu = {
diff --git a/arch/x86/oprofile/init.c b/arch/x86/oprofile/init.c
index cdfe4c54deca..f148cf652678 100644
--- a/arch/x86/oprofile/init.c
+++ b/arch/x86/oprofile/init.c
@@ -21,6 +21,7 @@ extern int op_nmi_timer_init(struct oprofile_operations *ops);
21extern void op_nmi_exit(void); 21extern void op_nmi_exit(void);
22extern void x86_backtrace(struct pt_regs * const regs, unsigned int depth); 22extern void x86_backtrace(struct pt_regs * const regs, unsigned int depth);
23 23
24static int nmi_timer;
24 25
25int __init oprofile_arch_init(struct oprofile_operations *ops) 26int __init oprofile_arch_init(struct oprofile_operations *ops)
26{ 27{
@@ -31,8 +32,9 @@ int __init oprofile_arch_init(struct oprofile_operations *ops)
31#ifdef CONFIG_X86_LOCAL_APIC 32#ifdef CONFIG_X86_LOCAL_APIC
32 ret = op_nmi_init(ops); 33 ret = op_nmi_init(ops);
33#endif 34#endif
35 nmi_timer = (ret != 0);
34#ifdef CONFIG_X86_IO_APIC 36#ifdef CONFIG_X86_IO_APIC
35 if (ret < 0) 37 if (nmi_timer)
36 ret = op_nmi_timer_init(ops); 38 ret = op_nmi_timer_init(ops);
37#endif 39#endif
38 ops->backtrace = x86_backtrace; 40 ops->backtrace = x86_backtrace;
@@ -44,6 +46,7 @@ int __init oprofile_arch_init(struct oprofile_operations *ops)
44void oprofile_arch_exit(void) 46void oprofile_arch_exit(void)
45{ 47{
46#ifdef CONFIG_X86_LOCAL_APIC 48#ifdef CONFIG_X86_LOCAL_APIC
47 op_nmi_exit(); 49 if (!nmi_timer)
50 op_nmi_exit();
48#endif 51#endif
49} 52}
diff --git a/drivers/oprofile/oprof.c b/drivers/oprofile/oprof.c
index dccd8636095c..f8c752e408a6 100644
--- a/drivers/oprofile/oprof.c
+++ b/drivers/oprofile/oprof.c
@@ -239,26 +239,45 @@ int oprofile_set_ulong(unsigned long *addr, unsigned long val)
239 return err; 239 return err;
240} 240}
241 241
242static int timer_mode;
243
242static int __init oprofile_init(void) 244static int __init oprofile_init(void)
243{ 245{
244 int err; 246 int err;
245 247
248 /* always init architecture to setup backtrace support */
246 err = oprofile_arch_init(&oprofile_ops); 249 err = oprofile_arch_init(&oprofile_ops);
247 if (err < 0 || timer) { 250
248 printk(KERN_INFO "oprofile: using timer interrupt.\n"); 251 timer_mode = err || timer; /* fall back to timer mode on errors */
252 if (timer_mode) {
253 if (!err)
254 oprofile_arch_exit();
249 err = oprofile_timer_init(&oprofile_ops); 255 err = oprofile_timer_init(&oprofile_ops);
250 if (err) 256 if (err)
251 return err; 257 return err;
252 } 258 }
253 return oprofilefs_register(); 259
260 err = oprofilefs_register();
261 if (!err)
262 return 0;
263
264 /* failed */
265 if (timer_mode)
266 oprofile_timer_exit();
267 else
268 oprofile_arch_exit();
269
270 return err;
254} 271}
255 272
256 273
257static void __exit oprofile_exit(void) 274static void __exit oprofile_exit(void)
258{ 275{
259 oprofile_timer_exit();
260 oprofilefs_unregister(); 276 oprofilefs_unregister();
261 oprofile_arch_exit(); 277 if (timer_mode)
278 oprofile_timer_exit();
279 else
280 oprofile_arch_exit();
262} 281}
263 282
264 283
diff --git a/drivers/oprofile/timer_int.c b/drivers/oprofile/timer_int.c
index 3ef44624f510..878fba126582 100644
--- a/drivers/oprofile/timer_int.c
+++ b/drivers/oprofile/timer_int.c
@@ -110,6 +110,7 @@ int oprofile_timer_init(struct oprofile_operations *ops)
110 ops->start = oprofile_hrtimer_start; 110 ops->start = oprofile_hrtimer_start;
111 ops->stop = oprofile_hrtimer_stop; 111 ops->stop = oprofile_hrtimer_stop;
112 ops->cpu_type = "timer"; 112 ops->cpu_type = "timer";
113 printk(KERN_INFO "oprofile: using timer interrupt.\n");
113 return 0; 114 return 0;
114} 115}
115 116
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 1e9ebe5e0091..b1f89122bf6a 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -822,6 +822,7 @@ struct perf_event {
822 int mmap_locked; 822 int mmap_locked;
823 struct user_struct *mmap_user; 823 struct user_struct *mmap_user;
824 struct ring_buffer *rb; 824 struct ring_buffer *rb;
825 struct list_head rb_entry;
825 826
826 /* poll related */ 827 /* poll related */
827 wait_queue_head_t waitq; 828 wait_queue_head_t waitq;
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 0e8457da6f95..600c1629b64d 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -185,6 +185,9 @@ static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
185static void update_context_time(struct perf_event_context *ctx); 185static void update_context_time(struct perf_event_context *ctx);
186static u64 perf_event_time(struct perf_event *event); 186static u64 perf_event_time(struct perf_event *event);
187 187
188static void ring_buffer_attach(struct perf_event *event,
189 struct ring_buffer *rb);
190
188void __weak perf_event_print_debug(void) { } 191void __weak perf_event_print_debug(void) { }
189 192
190extern __weak const char *perf_pmu_name(void) 193extern __weak const char *perf_pmu_name(void)
@@ -2173,7 +2176,8 @@ static void perf_event_context_sched_in(struct perf_event_context *ctx,
2173 2176
2174 perf_event_sched_in(cpuctx, ctx, task); 2177 perf_event_sched_in(cpuctx, ctx, task);
2175 2178
2176 cpuctx->task_ctx = ctx; 2179 if (ctx->nr_events)
2180 cpuctx->task_ctx = ctx;
2177 2181
2178 perf_pmu_enable(ctx->pmu); 2182 perf_pmu_enable(ctx->pmu);
2179 perf_ctx_unlock(cpuctx, ctx); 2183 perf_ctx_unlock(cpuctx, ctx);
@@ -3190,12 +3194,33 @@ static unsigned int perf_poll(struct file *file, poll_table *wait)
3190 struct ring_buffer *rb; 3194 struct ring_buffer *rb;
3191 unsigned int events = POLL_HUP; 3195 unsigned int events = POLL_HUP;
3192 3196
3197 /*
3198 * Race between perf_event_set_output() and perf_poll(): perf_poll()
3199 * grabs the rb reference but perf_event_set_output() overrides it.
3200 * Here is the timeline for two threads T1, T2:
3201 * t0: T1, rb = rcu_dereference(event->rb)
3202 * t1: T2, old_rb = event->rb
3203 * t2: T2, event->rb = new rb
3204 * t3: T2, ring_buffer_detach(old_rb)
3205 * t4: T1, ring_buffer_attach(rb1)
3206 * t5: T1, poll_wait(event->waitq)
3207 *
3208 * To avoid this problem, we grab mmap_mutex in perf_poll()
3209 * thereby ensuring that the assignment of the new ring buffer
3210 * and the detachment of the old buffer appear atomic to perf_poll()
3211 */
3212 mutex_lock(&event->mmap_mutex);
3213
3193 rcu_read_lock(); 3214 rcu_read_lock();
3194 rb = rcu_dereference(event->rb); 3215 rb = rcu_dereference(event->rb);
3195 if (rb) 3216 if (rb) {
3217 ring_buffer_attach(event, rb);
3196 events = atomic_xchg(&rb->poll, 0); 3218 events = atomic_xchg(&rb->poll, 0);
3219 }
3197 rcu_read_unlock(); 3220 rcu_read_unlock();
3198 3221
3222 mutex_unlock(&event->mmap_mutex);
3223
3199 poll_wait(file, &event->waitq, wait); 3224 poll_wait(file, &event->waitq, wait);
3200 3225
3201 return events; 3226 return events;
@@ -3496,6 +3521,49 @@ unlock:
3496 return ret; 3521 return ret;
3497} 3522}
3498 3523
3524static void ring_buffer_attach(struct perf_event *event,
3525 struct ring_buffer *rb)
3526{
3527 unsigned long flags;
3528
3529 if (!list_empty(&event->rb_entry))
3530 return;
3531
3532 spin_lock_irqsave(&rb->event_lock, flags);
3533 if (!list_empty(&event->rb_entry))
3534 goto unlock;
3535
3536 list_add(&event->rb_entry, &rb->event_list);
3537unlock:
3538 spin_unlock_irqrestore(&rb->event_lock, flags);
3539}
3540
3541static void ring_buffer_detach(struct perf_event *event,
3542 struct ring_buffer *rb)
3543{
3544 unsigned long flags;
3545
3546 if (list_empty(&event->rb_entry))
3547 return;
3548
3549 spin_lock_irqsave(&rb->event_lock, flags);
3550 list_del_init(&event->rb_entry);
3551 wake_up_all(&event->waitq);
3552 spin_unlock_irqrestore(&rb->event_lock, flags);
3553}
3554
3555static void ring_buffer_wakeup(struct perf_event *event)
3556{
3557 struct ring_buffer *rb;
3558
3559 rcu_read_lock();
3560 rb = rcu_dereference(event->rb);
3561 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
3562 wake_up_all(&event->waitq);
3563 }
3564 rcu_read_unlock();
3565}
3566
3499static void rb_free_rcu(struct rcu_head *rcu_head) 3567static void rb_free_rcu(struct rcu_head *rcu_head)
3500{ 3568{
3501 struct ring_buffer *rb; 3569 struct ring_buffer *rb;
@@ -3521,9 +3589,19 @@ static struct ring_buffer *ring_buffer_get(struct perf_event *event)
3521 3589
3522static void ring_buffer_put(struct ring_buffer *rb) 3590static void ring_buffer_put(struct ring_buffer *rb)
3523{ 3591{
3592 struct perf_event *event, *n;
3593 unsigned long flags;
3594
3524 if (!atomic_dec_and_test(&rb->refcount)) 3595 if (!atomic_dec_and_test(&rb->refcount))
3525 return; 3596 return;
3526 3597
3598 spin_lock_irqsave(&rb->event_lock, flags);
3599 list_for_each_entry_safe(event, n, &rb->event_list, rb_entry) {
3600 list_del_init(&event->rb_entry);
3601 wake_up_all(&event->waitq);
3602 }
3603 spin_unlock_irqrestore(&rb->event_lock, flags);
3604
3527 call_rcu(&rb->rcu_head, rb_free_rcu); 3605 call_rcu(&rb->rcu_head, rb_free_rcu);
3528} 3606}
3529 3607
@@ -3546,6 +3624,7 @@ static void perf_mmap_close(struct vm_area_struct *vma)
3546 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm); 3624 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3547 vma->vm_mm->pinned_vm -= event->mmap_locked; 3625 vma->vm_mm->pinned_vm -= event->mmap_locked;
3548 rcu_assign_pointer(event->rb, NULL); 3626 rcu_assign_pointer(event->rb, NULL);
3627 ring_buffer_detach(event, rb);
3549 mutex_unlock(&event->mmap_mutex); 3628 mutex_unlock(&event->mmap_mutex);
3550 3629
3551 ring_buffer_put(rb); 3630 ring_buffer_put(rb);
@@ -3700,7 +3779,7 @@ static const struct file_operations perf_fops = {
3700 3779
3701void perf_event_wakeup(struct perf_event *event) 3780void perf_event_wakeup(struct perf_event *event)
3702{ 3781{
3703 wake_up_all(&event->waitq); 3782 ring_buffer_wakeup(event);
3704 3783
3705 if (event->pending_kill) { 3784 if (event->pending_kill) {
3706 kill_fasync(&event->fasync, SIGIO, event->pending_kill); 3785 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
@@ -5822,6 +5901,8 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
5822 INIT_LIST_HEAD(&event->group_entry); 5901 INIT_LIST_HEAD(&event->group_entry);
5823 INIT_LIST_HEAD(&event->event_entry); 5902 INIT_LIST_HEAD(&event->event_entry);
5824 INIT_LIST_HEAD(&event->sibling_list); 5903 INIT_LIST_HEAD(&event->sibling_list);
5904 INIT_LIST_HEAD(&event->rb_entry);
5905
5825 init_waitqueue_head(&event->waitq); 5906 init_waitqueue_head(&event->waitq);
5826 init_irq_work(&event->pending, perf_pending_event); 5907 init_irq_work(&event->pending, perf_pending_event);
5827 5908
@@ -6028,6 +6109,8 @@ set:
6028 6109
6029 old_rb = event->rb; 6110 old_rb = event->rb;
6030 rcu_assign_pointer(event->rb, rb); 6111 rcu_assign_pointer(event->rb, rb);
6112 if (old_rb)
6113 ring_buffer_detach(event, old_rb);
6031 ret = 0; 6114 ret = 0;
6032unlock: 6115unlock:
6033 mutex_unlock(&event->mmap_mutex); 6116 mutex_unlock(&event->mmap_mutex);
diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index 09097dd8116c..64568a699375 100644
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -22,6 +22,9 @@ struct ring_buffer {
22 local_t lost; /* nr records lost */ 22 local_t lost; /* nr records lost */
23 23
24 long watermark; /* wakeup watermark */ 24 long watermark; /* wakeup watermark */
25 /* poll crap */
26 spinlock_t event_lock;
27 struct list_head event_list;
25 28
26 struct perf_event_mmap_page *user_page; 29 struct perf_event_mmap_page *user_page;
27 void *data_pages[0]; 30 void *data_pages[0];
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index a2a29205cc0f..7f3011c6b57f 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -209,6 +209,9 @@ ring_buffer_init(struct ring_buffer *rb, long watermark, int flags)
209 rb->writable = 1; 209 rb->writable = 1;
210 210
211 atomic_set(&rb->refcount, 1); 211 atomic_set(&rb->refcount, 1);
212
213 INIT_LIST_HEAD(&rb->event_list);
214 spin_lock_init(&rb->event_lock);
212} 215}
213 216
214#ifndef CONFIG_PERF_USE_VMALLOC 217#ifndef CONFIG_PERF_USE_VMALLOC
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index 816d3d074979..d6e7926dcd26 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -1686,7 +1686,7 @@ static int replace_system_preds(struct event_subsystem *system,
1686 * replace the filter for the call. 1686 * replace the filter for the call.
1687 */ 1687 */
1688 filter = call->filter; 1688 filter = call->filter;
1689 call->filter = filter_item->filter; 1689 rcu_assign_pointer(call->filter, filter_item->filter);
1690 filter_item->filter = filter; 1690 filter_item->filter = filter;
1691 1691
1692 fail = false; 1692 fail = false;
@@ -1741,7 +1741,7 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string)
1741 filter = call->filter; 1741 filter = call->filter;
1742 if (!filter) 1742 if (!filter)
1743 goto out_unlock; 1743 goto out_unlock;
1744 call->filter = NULL; 1744 RCU_INIT_POINTER(call->filter, NULL);
1745 /* Make sure the filter is not being used */ 1745 /* Make sure the filter is not being used */
1746 synchronize_sched(); 1746 synchronize_sched();
1747 __free_filter(filter); 1747 __free_filter(filter);
@@ -1782,7 +1782,7 @@ out:
1782 * string 1782 * string
1783 */ 1783 */
1784 tmp = call->filter; 1784 tmp = call->filter;
1785 call->filter = filter; 1785 rcu_assign_pointer(call->filter, filter);
1786 if (tmp) { 1786 if (tmp) {
1787 /* Make sure the call is done with the filter */ 1787 /* Make sure the call is done with the filter */
1788 synchronize_sched(); 1788 synchronize_sched();
diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c
index e42626422587..d7915d4e77cb 100644
--- a/tools/perf/util/evsel.c
+++ b/tools/perf/util/evsel.c
@@ -34,6 +34,16 @@ int __perf_evsel__sample_size(u64 sample_type)
34 return size; 34 return size;
35} 35}
36 36
37static void hists__init(struct hists *hists)
38{
39 memset(hists, 0, sizeof(*hists));
40 hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT;
41 hists->entries_in = &hists->entries_in_array[0];
42 hists->entries_collapsed = RB_ROOT;
43 hists->entries = RB_ROOT;
44 pthread_mutex_init(&hists->lock, NULL);
45}
46
37void perf_evsel__init(struct perf_evsel *evsel, 47void perf_evsel__init(struct perf_evsel *evsel,
38 struct perf_event_attr *attr, int idx) 48 struct perf_event_attr *attr, int idx)
39{ 49{
diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c
index a36a3fa81ffb..abef2703cd24 100644
--- a/tools/perf/util/hist.c
+++ b/tools/perf/util/hist.c
@@ -1211,13 +1211,3 @@ size_t hists__fprintf_nr_events(struct hists *hists, FILE *fp)
1211 1211
1212 return ret; 1212 return ret;
1213} 1213}
1214
1215void hists__init(struct hists *hists)
1216{
1217 memset(hists, 0, sizeof(*hists));
1218 hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT;
1219 hists->entries_in = &hists->entries_in_array[0];
1220 hists->entries_collapsed = RB_ROOT;
1221 hists->entries = RB_ROOT;
1222 pthread_mutex_init(&hists->lock, NULL);
1223}
diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h
index c86c1d27bd1e..89289c8e935e 100644
--- a/tools/perf/util/hist.h
+++ b/tools/perf/util/hist.h
@@ -63,8 +63,6 @@ struct hists {
63 struct callchain_cursor callchain_cursor; 63 struct callchain_cursor callchain_cursor;
64}; 64};
65 65
66void hists__init(struct hists *hists);
67
68struct hist_entry *__hists__add_entry(struct hists *self, 66struct hist_entry *__hists__add_entry(struct hists *self,
69 struct addr_location *al, 67 struct addr_location *al,
70 struct symbol *parent, u64 period); 68 struct symbol *parent, u64 period);
diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c
index 85c1e6b76f0a..0f4555ce9063 100644
--- a/tools/perf/util/session.c
+++ b/tools/perf/util/session.c
@@ -1333,6 +1333,10 @@ int perf_session__cpu_bitmap(struct perf_session *session,
1333 } 1333 }
1334 1334
1335 map = cpu_map__new(cpu_list); 1335 map = cpu_map__new(cpu_list);
1336 if (map == NULL) {
1337 pr_err("Invalid cpu_list\n");
1338 return -1;
1339 }
1336 1340
1337 for (i = 0; i < map->nr; i++) { 1341 for (i = 0; i < map->nr; i++) {
1338 int cpu = map->map[i]; 1342 int cpu = map->map[i];