aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/perf_event.c
diff options
context:
space:
mode:
authorPeter Zijlstra <a.p.zijlstra@chello.nl>2010-09-06 08:32:03 -0400
committerIngo Molnar <mingo@elte.hu>2010-09-09 14:46:32 -0400
commit108b02cfce04ee90b0a07ee0b104baffd39f5934 (patch)
treee209d5fe01cda90df2cc0241c6247fd934cc5087 /kernel/perf_event.c
parentb5ab4cd563e7ab49b27957704112a8ecade54e1f (diff)
perf: Per-pmu-per-cpu contexts
Allocate per-cpu contexts per pmu. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: paulus <paulus@samba.org> Cc: stephane eranian <eranian@googlemail.com> Cc: Robert Richter <robert.richter@amd.com> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Lin Ming <ming.m.lin@intel.com> Cc: Yanmin <yanmin_zhang@linux.intel.com> LKML-Reference: <new-submission> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel/perf_event.c')
-rw-r--r--kernel/perf_event.c178
1 files changed, 110 insertions, 68 deletions
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index d75e4c8727f9..8ca6e690ffe3 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -34,16 +34,15 @@
34 34
35#include <asm/irq_regs.h> 35#include <asm/irq_regs.h>
36 36
37/*
38 * Each CPU has a list of per CPU events:
39 */
40static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
41
42static atomic_t nr_events __read_mostly; 37static atomic_t nr_events __read_mostly;
43static atomic_t nr_mmap_events __read_mostly; 38static atomic_t nr_mmap_events __read_mostly;
44static atomic_t nr_comm_events __read_mostly; 39static atomic_t nr_comm_events __read_mostly;
45static atomic_t nr_task_events __read_mostly; 40static atomic_t nr_task_events __read_mostly;
46 41
42static LIST_HEAD(pmus);
43static DEFINE_MUTEX(pmus_lock);
44static struct srcu_struct pmus_srcu;
45
47/* 46/*
48 * perf event paranoia level: 47 * perf event paranoia level:
49 * -1 - not paranoid at all 48 * -1 - not paranoid at all
@@ -78,9 +77,9 @@ void perf_pmu_enable(struct pmu *pmu)
78 pmu->pmu_enable(pmu); 77 pmu->pmu_enable(pmu);
79} 78}
80 79
81static void perf_pmu_rotate_start(void) 80static void perf_pmu_rotate_start(struct pmu *pmu)
82{ 81{
83 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); 82 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
84 83
85 if (hrtimer_active(&cpuctx->timer)) 84 if (hrtimer_active(&cpuctx->timer))
86 return; 85 return;
@@ -90,9 +89,9 @@ static void perf_pmu_rotate_start(void)
90 HRTIMER_MODE_REL_PINNED, 0); 89 HRTIMER_MODE_REL_PINNED, 0);
91} 90}
92 91
93static void perf_pmu_rotate_stop(void) 92static void perf_pmu_rotate_stop(struct pmu *pmu)
94{ 93{
95 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); 94 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
96 95
97 hrtimer_cancel(&cpuctx->timer); 96 hrtimer_cancel(&cpuctx->timer);
98} 97}
@@ -301,7 +300,7 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx)
301 300
302 list_add_rcu(&event->event_entry, &ctx->event_list); 301 list_add_rcu(&event->event_entry, &ctx->event_list);
303 if (!ctx->nr_events) 302 if (!ctx->nr_events)
304 perf_pmu_rotate_start(); 303 perf_pmu_rotate_start(ctx->pmu);
305 ctx->nr_events++; 304 ctx->nr_events++;
306 if (event->attr.inherit_stat) 305 if (event->attr.inherit_stat)
307 ctx->nr_stat++; 306 ctx->nr_stat++;
@@ -466,6 +465,12 @@ group_sched_out(struct perf_event *group_event,
466 cpuctx->exclusive = 0; 465 cpuctx->exclusive = 0;
467} 466}
468 467
468static inline struct perf_cpu_context *
469__get_cpu_context(struct perf_event_context *ctx)
470{
471 return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
472}
473
469/* 474/*
470 * Cross CPU call to remove a performance event 475 * Cross CPU call to remove a performance event
471 * 476 *
@@ -474,9 +479,9 @@ group_sched_out(struct perf_event *group_event,
474 */ 479 */
475static void __perf_event_remove_from_context(void *info) 480static void __perf_event_remove_from_context(void *info)
476{ 481{
477 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
478 struct perf_event *event = info; 482 struct perf_event *event = info;
479 struct perf_event_context *ctx = event->ctx; 483 struct perf_event_context *ctx = event->ctx;
484 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
480 485
481 /* 486 /*
482 * If this is a task context, we need to check whether it is 487 * If this is a task context, we need to check whether it is
@@ -556,8 +561,8 @@ retry:
556static void __perf_event_disable(void *info) 561static void __perf_event_disable(void *info)
557{ 562{
558 struct perf_event *event = info; 563 struct perf_event *event = info;
559 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
560 struct perf_event_context *ctx = event->ctx; 564 struct perf_event_context *ctx = event->ctx;
565 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
561 566
562 /* 567 /*
563 * If this is a per-task event, need to check whether this 568 * If this is a per-task event, need to check whether this
@@ -765,10 +770,10 @@ static void add_event_to_ctx(struct perf_event *event,
765 */ 770 */
766static void __perf_install_in_context(void *info) 771static void __perf_install_in_context(void *info)
767{ 772{
768 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
769 struct perf_event *event = info; 773 struct perf_event *event = info;
770 struct perf_event_context *ctx = event->ctx; 774 struct perf_event_context *ctx = event->ctx;
771 struct perf_event *leader = event->group_leader; 775 struct perf_event *leader = event->group_leader;
776 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
772 int err; 777 int err;
773 778
774 /* 779 /*
@@ -912,9 +917,9 @@ static void __perf_event_mark_enabled(struct perf_event *event,
912static void __perf_event_enable(void *info) 917static void __perf_event_enable(void *info)
913{ 918{
914 struct perf_event *event = info; 919 struct perf_event *event = info;
915 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
916 struct perf_event_context *ctx = event->ctx; 920 struct perf_event_context *ctx = event->ctx;
917 struct perf_event *leader = event->group_leader; 921 struct perf_event *leader = event->group_leader;
922 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
918 int err; 923 int err;
919 924
920 /* 925 /*
@@ -1188,15 +1193,19 @@ static void perf_event_sync_stat(struct perf_event_context *ctx,
1188void perf_event_task_sched_out(struct task_struct *task, 1193void perf_event_task_sched_out(struct task_struct *task,
1189 struct task_struct *next) 1194 struct task_struct *next)
1190{ 1195{
1191 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1192 struct perf_event_context *ctx = task->perf_event_ctxp; 1196 struct perf_event_context *ctx = task->perf_event_ctxp;
1193 struct perf_event_context *next_ctx; 1197 struct perf_event_context *next_ctx;
1194 struct perf_event_context *parent; 1198 struct perf_event_context *parent;
1199 struct perf_cpu_context *cpuctx;
1195 int do_switch = 1; 1200 int do_switch = 1;
1196 1201
1197 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0); 1202 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
1198 1203
1199 if (likely(!ctx || !cpuctx->task_ctx)) 1204 if (likely(!ctx))
1205 return;
1206
1207 cpuctx = __get_cpu_context(ctx);
1208 if (!cpuctx->task_ctx)
1200 return; 1209 return;
1201 1210
1202 rcu_read_lock(); 1211 rcu_read_lock();
@@ -1242,7 +1251,7 @@ void perf_event_task_sched_out(struct task_struct *task,
1242static void task_ctx_sched_out(struct perf_event_context *ctx, 1251static void task_ctx_sched_out(struct perf_event_context *ctx,
1243 enum event_type_t event_type) 1252 enum event_type_t event_type)
1244{ 1253{
1245 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); 1254 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1246 1255
1247 if (!cpuctx->task_ctx) 1256 if (!cpuctx->task_ctx)
1248 return; 1257 return;
@@ -1360,8 +1369,8 @@ static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
1360static void task_ctx_sched_in(struct task_struct *task, 1369static void task_ctx_sched_in(struct task_struct *task,
1361 enum event_type_t event_type) 1370 enum event_type_t event_type)
1362{ 1371{
1363 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1364 struct perf_event_context *ctx = task->perf_event_ctxp; 1372 struct perf_event_context *ctx = task->perf_event_ctxp;
1373 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1365 1374
1366 if (likely(!ctx)) 1375 if (likely(!ctx))
1367 return; 1376 return;
@@ -1383,12 +1392,13 @@ static void task_ctx_sched_in(struct task_struct *task,
1383 */ 1392 */
1384void perf_event_task_sched_in(struct task_struct *task) 1393void perf_event_task_sched_in(struct task_struct *task)
1385{ 1394{
1386 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1387 struct perf_event_context *ctx = task->perf_event_ctxp; 1395 struct perf_event_context *ctx = task->perf_event_ctxp;
1396 struct perf_cpu_context *cpuctx;
1388 1397
1389 if (likely(!ctx)) 1398 if (likely(!ctx))
1390 return; 1399 return;
1391 1400
1401 cpuctx = __get_cpu_context(ctx);
1392 if (cpuctx->task_ctx == ctx) 1402 if (cpuctx->task_ctx == ctx)
1393 return; 1403 return;
1394 1404
@@ -1409,7 +1419,7 @@ void perf_event_task_sched_in(struct task_struct *task)
1409 * Since these rotations are per-cpu, we need to ensure the 1419 * Since these rotations are per-cpu, we need to ensure the
1410 * cpu-context we got scheduled on is actually rotating. 1420 * cpu-context we got scheduled on is actually rotating.
1411 */ 1421 */
1412 perf_pmu_rotate_start(); 1422 perf_pmu_rotate_start(ctx->pmu);
1413} 1423}
1414 1424
1415#define MAX_INTERRUPTS (~0ULL) 1425#define MAX_INTERRUPTS (~0ULL)
@@ -1687,9 +1697,9 @@ out:
1687 */ 1697 */
1688static void __perf_event_read(void *info) 1698static void __perf_event_read(void *info)
1689{ 1699{
1690 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1691 struct perf_event *event = info; 1700 struct perf_event *event = info;
1692 struct perf_event_context *ctx = event->ctx; 1701 struct perf_event_context *ctx = event->ctx;
1702 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1693 1703
1694 /* 1704 /*
1695 * If this is a task context, we need to check whether it is 1705 * If this is a task context, we need to check whether it is
@@ -1962,7 +1972,8 @@ __perf_event_init_context(struct perf_event_context *ctx,
1962 ctx->task = task; 1972 ctx->task = task;
1963} 1973}
1964 1974
1965static struct perf_event_context *find_get_context(pid_t pid, int cpu) 1975static struct perf_event_context *
1976find_get_context(struct pmu *pmu, pid_t pid, int cpu)
1966{ 1977{
1967 struct perf_event_context *ctx; 1978 struct perf_event_context *ctx;
1968 struct perf_cpu_context *cpuctx; 1979 struct perf_cpu_context *cpuctx;
@@ -1986,7 +1997,7 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu)
1986 if (!cpu_online(cpu)) 1997 if (!cpu_online(cpu))
1987 return ERR_PTR(-ENODEV); 1998 return ERR_PTR(-ENODEV);
1988 1999
1989 cpuctx = &per_cpu(perf_cpu_context, cpu); 2000 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
1990 ctx = &cpuctx->ctx; 2001 ctx = &cpuctx->ctx;
1991 get_ctx(ctx); 2002 get_ctx(ctx);
1992 2003
@@ -2030,6 +2041,7 @@ retry:
2030 if (!ctx) 2041 if (!ctx)
2031 goto errout; 2042 goto errout;
2032 __perf_event_init_context(ctx, task); 2043 __perf_event_init_context(ctx, task);
2044 ctx->pmu = pmu;
2033 get_ctx(ctx); 2045 get_ctx(ctx);
2034 if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) { 2046 if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
2035 /* 2047 /*
@@ -3745,18 +3757,20 @@ static void perf_event_task_ctx(struct perf_event_context *ctx,
3745 3757
3746static void perf_event_task_event(struct perf_task_event *task_event) 3758static void perf_event_task_event(struct perf_task_event *task_event)
3747{ 3759{
3748 struct perf_cpu_context *cpuctx;
3749 struct perf_event_context *ctx = task_event->task_ctx; 3760 struct perf_event_context *ctx = task_event->task_ctx;
3761 struct perf_cpu_context *cpuctx;
3762 struct pmu *pmu;
3750 3763
3751 rcu_read_lock(); 3764 rcu_read_lock_sched();
3752 cpuctx = &get_cpu_var(perf_cpu_context); 3765 list_for_each_entry_rcu(pmu, &pmus, entry) {
3753 perf_event_task_ctx(&cpuctx->ctx, task_event); 3766 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
3767 perf_event_task_ctx(&cpuctx->ctx, task_event);
3768 }
3754 if (!ctx) 3769 if (!ctx)
3755 ctx = rcu_dereference(current->perf_event_ctxp); 3770 ctx = rcu_dereference(current->perf_event_ctxp);
3756 if (ctx) 3771 if (ctx)
3757 perf_event_task_ctx(ctx, task_event); 3772 perf_event_task_ctx(ctx, task_event);
3758 put_cpu_var(perf_cpu_context); 3773 rcu_read_unlock_sched();
3759 rcu_read_unlock();
3760} 3774}
3761 3775
3762static void perf_event_task(struct task_struct *task, 3776static void perf_event_task(struct task_struct *task,
@@ -3861,6 +3875,7 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
3861 struct perf_cpu_context *cpuctx; 3875 struct perf_cpu_context *cpuctx;
3862 struct perf_event_context *ctx; 3876 struct perf_event_context *ctx;
3863 unsigned int size; 3877 unsigned int size;
3878 struct pmu *pmu;
3864 char comm[TASK_COMM_LEN]; 3879 char comm[TASK_COMM_LEN];
3865 3880
3866 memset(comm, 0, sizeof(comm)); 3881 memset(comm, 0, sizeof(comm));
@@ -3872,14 +3887,15 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
3872 3887
3873 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; 3888 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3874 3889
3875 rcu_read_lock(); 3890 rcu_read_lock_sched();
3876 cpuctx = &get_cpu_var(perf_cpu_context); 3891 list_for_each_entry_rcu(pmu, &pmus, entry) {
3877 perf_event_comm_ctx(&cpuctx->ctx, comm_event); 3892 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
3893 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
3894 }
3878 ctx = rcu_dereference(current->perf_event_ctxp); 3895 ctx = rcu_dereference(current->perf_event_ctxp);
3879 if (ctx) 3896 if (ctx)
3880 perf_event_comm_ctx(ctx, comm_event); 3897 perf_event_comm_ctx(ctx, comm_event);
3881 put_cpu_var(perf_cpu_context); 3898 rcu_read_unlock_sched();
3882 rcu_read_unlock();
3883} 3899}
3884 3900
3885void perf_event_comm(struct task_struct *task) 3901void perf_event_comm(struct task_struct *task)
@@ -3989,6 +4005,7 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3989 char tmp[16]; 4005 char tmp[16];
3990 char *buf = NULL; 4006 char *buf = NULL;
3991 const char *name; 4007 const char *name;
4008 struct pmu *pmu;
3992 4009
3993 memset(tmp, 0, sizeof(tmp)); 4010 memset(tmp, 0, sizeof(tmp));
3994 4011
@@ -4040,14 +4057,16 @@ got_name:
4040 4057
4041 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; 4058 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4042 4059
4043 rcu_read_lock(); 4060 rcu_read_lock_sched();
4044 cpuctx = &get_cpu_var(perf_cpu_context); 4061 list_for_each_entry_rcu(pmu, &pmus, entry) {
4045 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event, vma->vm_flags & VM_EXEC); 4062 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
4063 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4064 vma->vm_flags & VM_EXEC);
4065 }
4046 ctx = rcu_dereference(current->perf_event_ctxp); 4066 ctx = rcu_dereference(current->perf_event_ctxp);
4047 if (ctx) 4067 if (ctx)
4048 perf_event_mmap_ctx(ctx, mmap_event, vma->vm_flags & VM_EXEC); 4068 perf_event_mmap_ctx(ctx, mmap_event, vma->vm_flags & VM_EXEC);
4049 put_cpu_var(perf_cpu_context); 4069 rcu_read_unlock_sched();
4050 rcu_read_unlock();
4051 4070
4052 kfree(buf); 4071 kfree(buf);
4053} 4072}
@@ -4982,10 +5001,6 @@ static struct pmu perf_task_clock = {
4982 .read = task_clock_event_read, 5001 .read = task_clock_event_read,
4983}; 5002};
4984 5003
4985static LIST_HEAD(pmus);
4986static DEFINE_MUTEX(pmus_lock);
4987static struct srcu_struct pmus_srcu;
4988
4989static void perf_pmu_nop_void(struct pmu *pmu) 5004static void perf_pmu_nop_void(struct pmu *pmu)
4990{ 5005{
4991} 5006}
@@ -5013,7 +5028,7 @@ static void perf_pmu_cancel_txn(struct pmu *pmu)
5013 5028
5014int perf_pmu_register(struct pmu *pmu) 5029int perf_pmu_register(struct pmu *pmu)
5015{ 5030{
5016 int ret; 5031 int cpu, ret;
5017 5032
5018 mutex_lock(&pmus_lock); 5033 mutex_lock(&pmus_lock);
5019 ret = -ENOMEM; 5034 ret = -ENOMEM;
@@ -5021,6 +5036,21 @@ int perf_pmu_register(struct pmu *pmu)
5021 if (!pmu->pmu_disable_count) 5036 if (!pmu->pmu_disable_count)
5022 goto unlock; 5037 goto unlock;
5023 5038
5039 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
5040 if (!pmu->pmu_cpu_context)
5041 goto free_pdc;
5042
5043 for_each_possible_cpu(cpu) {
5044 struct perf_cpu_context *cpuctx;
5045
5046 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5047 __perf_event_init_context(&cpuctx->ctx, NULL);
5048 cpuctx->ctx.pmu = pmu;
5049 cpuctx->timer_interval = TICK_NSEC;
5050 hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5051 cpuctx->timer.function = perf_event_context_tick;
5052 }
5053
5024 if (!pmu->start_txn) { 5054 if (!pmu->start_txn) {
5025 if (pmu->pmu_enable) { 5055 if (pmu->pmu_enable) {
5026 /* 5056 /*
@@ -5049,6 +5079,10 @@ unlock:
5049 mutex_unlock(&pmus_lock); 5079 mutex_unlock(&pmus_lock);
5050 5080
5051 return ret; 5081 return ret;
5082
5083free_pdc:
5084 free_percpu(pmu->pmu_disable_count);
5085 goto unlock;
5052} 5086}
5053 5087
5054void perf_pmu_unregister(struct pmu *pmu) 5088void perf_pmu_unregister(struct pmu *pmu)
@@ -5057,9 +5091,14 @@ void perf_pmu_unregister(struct pmu *pmu)
5057 list_del_rcu(&pmu->entry); 5091 list_del_rcu(&pmu->entry);
5058 mutex_unlock(&pmus_lock); 5092 mutex_unlock(&pmus_lock);
5059 5093
5094 /*
5095 * We use the pmu list either under SRCU or preempt_disable,
5096 * synchronize_srcu() implies synchronize_sched() so we're good.
5097 */
5060 synchronize_srcu(&pmus_srcu); 5098 synchronize_srcu(&pmus_srcu);
5061 5099
5062 free_percpu(pmu->pmu_disable_count); 5100 free_percpu(pmu->pmu_disable_count);
5101 free_percpu(pmu->pmu_cpu_context);
5063} 5102}
5064 5103
5065struct pmu *perf_init_event(struct perf_event *event) 5104struct pmu *perf_init_event(struct perf_event *event)
@@ -5374,7 +5413,7 @@ SYSCALL_DEFINE5(perf_event_open,
5374 /* 5413 /*
5375 * Get the target context (task or percpu): 5414 * Get the target context (task or percpu):
5376 */ 5415 */
5377 ctx = find_get_context(pid, cpu); 5416 ctx = find_get_context(event->pmu, pid, cpu);
5378 if (IS_ERR(ctx)) { 5417 if (IS_ERR(ctx)) {
5379 err = PTR_ERR(ctx); 5418 err = PTR_ERR(ctx);
5380 goto err_alloc; 5419 goto err_alloc;
@@ -5489,7 +5528,7 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
5489 goto err; 5528 goto err;
5490 } 5529 }
5491 5530
5492 ctx = find_get_context(pid, cpu); 5531 ctx = find_get_context(event->pmu, pid, cpu);
5493 if (IS_ERR(ctx)) { 5532 if (IS_ERR(ctx)) {
5494 err = PTR_ERR(ctx); 5533 err = PTR_ERR(ctx);
5495 goto err_free; 5534 goto err_free;
@@ -5833,6 +5872,7 @@ inherit_task_group(struct perf_event *event, struct task_struct *parent,
5833 return -ENOMEM; 5872 return -ENOMEM;
5834 5873
5835 __perf_event_init_context(child_ctx, child); 5874 __perf_event_init_context(child_ctx, child);
5875 child_ctx->pmu = event->pmu;
5836 child->perf_event_ctxp = child_ctx; 5876 child->perf_event_ctxp = child_ctx;
5837 get_task_struct(child); 5877 get_task_struct(child);
5838 } 5878 }
@@ -5935,30 +5975,18 @@ int perf_event_init_task(struct task_struct *child)
5935 5975
5936static void __init perf_event_init_all_cpus(void) 5976static void __init perf_event_init_all_cpus(void)
5937{ 5977{
5938 struct perf_cpu_context *cpuctx;
5939 struct swevent_htable *swhash; 5978 struct swevent_htable *swhash;
5940 int cpu; 5979 int cpu;
5941 5980
5942 for_each_possible_cpu(cpu) { 5981 for_each_possible_cpu(cpu) {
5943 swhash = &per_cpu(swevent_htable, cpu); 5982 swhash = &per_cpu(swevent_htable, cpu);
5944 mutex_init(&swhash->hlist_mutex); 5983 mutex_init(&swhash->hlist_mutex);
5945
5946 cpuctx = &per_cpu(perf_cpu_context, cpu);
5947 __perf_event_init_context(&cpuctx->ctx, NULL);
5948 cpuctx->timer_interval = TICK_NSEC;
5949 hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5950 cpuctx->timer.function = perf_event_context_tick;
5951 } 5984 }
5952} 5985}
5953 5986
5954static void __cpuinit perf_event_init_cpu(int cpu) 5987static void __cpuinit perf_event_init_cpu(int cpu)
5955{ 5988{
5956 struct perf_cpu_context *cpuctx; 5989 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5957 struct swevent_htable *swhash;
5958
5959 cpuctx = &per_cpu(perf_cpu_context, cpu);
5960
5961 swhash = &per_cpu(swevent_htable, cpu);
5962 5990
5963 mutex_lock(&swhash->hlist_mutex); 5991 mutex_lock(&swhash->hlist_mutex);
5964 if (swhash->hlist_refcount > 0) { 5992 if (swhash->hlist_refcount > 0) {
@@ -5972,32 +6000,46 @@ static void __cpuinit perf_event_init_cpu(int cpu)
5972} 6000}
5973 6001
5974#ifdef CONFIG_HOTPLUG_CPU 6002#ifdef CONFIG_HOTPLUG_CPU
5975static void __perf_event_exit_cpu(void *info) 6003static void __perf_event_exit_context(void *__info)
5976{ 6004{
5977 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); 6005 struct perf_event_context *ctx = __info;
5978 struct perf_event_context *ctx = &cpuctx->ctx;
5979 struct perf_event *event, *tmp; 6006 struct perf_event *event, *tmp;
5980 6007
5981 perf_pmu_rotate_stop(); 6008 perf_pmu_rotate_stop(ctx->pmu);
5982 6009
5983 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry) 6010 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
5984 __perf_event_remove_from_context(event); 6011 __perf_event_remove_from_context(event);
5985 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry) 6012 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
5986 __perf_event_remove_from_context(event); 6013 __perf_event_remove_from_context(event);
5987} 6014}
6015
6016static void perf_event_exit_cpu_context(int cpu)
6017{
6018 struct perf_event_context *ctx;
6019 struct pmu *pmu;
6020 int idx;
6021
6022 idx = srcu_read_lock(&pmus_srcu);
6023 list_for_each_entry_rcu(pmu, &pmus, entry) {
6024 ctx = &this_cpu_ptr(pmu->pmu_cpu_context)->ctx;
6025
6026 mutex_lock(&ctx->mutex);
6027 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
6028 mutex_unlock(&ctx->mutex);
6029 }
6030 srcu_read_unlock(&pmus_srcu, idx);
6031
6032}
6033
5988static void perf_event_exit_cpu(int cpu) 6034static void perf_event_exit_cpu(int cpu)
5989{ 6035{
5990 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5991 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 6036 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5992 struct perf_event_context *ctx = &cpuctx->ctx;
5993 6037
5994 mutex_lock(&swhash->hlist_mutex); 6038 mutex_lock(&swhash->hlist_mutex);
5995 swevent_hlist_release(swhash); 6039 swevent_hlist_release(swhash);
5996 mutex_unlock(&swhash->hlist_mutex); 6040 mutex_unlock(&swhash->hlist_mutex);
5997 6041
5998 mutex_lock(&ctx->mutex); 6042 perf_event_exit_cpu_context(cpu);
5999 smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
6000 mutex_unlock(&ctx->mutex);
6001} 6043}
6002#else 6044#else
6003static inline void perf_event_exit_cpu(int cpu) { } 6045static inline void perf_event_exit_cpu(int cpu) { }