aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorFrederic Weisbecker <fweisbec@gmail.com>2010-05-20 04:17:46 -0400
committerFrederic Weisbecker <fweisbec@gmail.com>2010-05-20 04:40:37 -0400
commit49f135ed02828a58b2401f149926c2e3c9cb0116 (patch)
tree6f5541da5ad60131eb32beb42083443705351750
parentcbb5cf7ff6b298beacfe23db3386335b0b9c0a2d (diff)
perf: Comply with new rcu checks API
The software events hlist doesn't fully comply with the new rcu checks api. We need to consider three different sides that access the hlist: - the hlist allocation/release side. This side happens when an events is created or released, accesses to the hlist are serialized under the cpuctx mutex. - the events insertion/removal in the hlist. This side is always serialized against the above one. The hlist is always present during such operations. This side happens when a software event is scheduled in/out. The serialization that ensures the software event is really attached to the context is made under the ctx->lock. - events triggering. This is the read side, it can happen concurrently with any update side. This patch deals with them one by one and anticipates with the separate rcu mem space patches in preparation. This patch fixes various annoying rcu warnings. Reported-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Paul Mackerras <paulus@samba.org>
-rw-r--r--kernel/perf_event.c58
1 files changed, 46 insertions, 12 deletions
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index a4fa381db3c2..511677bc1c6a 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -4066,19 +4066,46 @@ static inline u64 swevent_hash(u64 type, u32 event_id)
4066 return hash_64(val, SWEVENT_HLIST_BITS); 4066 return hash_64(val, SWEVENT_HLIST_BITS);
4067} 4067}
4068 4068
4069static struct hlist_head * 4069static inline struct hlist_head *
4070find_swevent_head(struct perf_cpu_context *ctx, u64 type, u32 event_id) 4070__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4071{ 4071{
4072 u64 hash; 4072 u64 hash = swevent_hash(type, event_id);
4073 struct swevent_hlist *hlist; 4073
4074 return &hlist->heads[hash];
4075}
4074 4076
4075 hash = swevent_hash(type, event_id); 4077/* For the read side: events when they trigger */
4078static inline struct hlist_head *
4079find_swevent_head_rcu(struct perf_cpu_context *ctx, u64 type, u32 event_id)
4080{
4081 struct swevent_hlist *hlist;
4076 4082
4077 hlist = rcu_dereference(ctx->swevent_hlist); 4083 hlist = rcu_dereference(ctx->swevent_hlist);
4078 if (!hlist) 4084 if (!hlist)
4079 return NULL; 4085 return NULL;
4080 4086
4081 return &hlist->heads[hash]; 4087 return __find_swevent_head(hlist, type, event_id);
4088}
4089
4090/* For the event head insertion and removal in the hlist */
4091static inline struct hlist_head *
4092find_swevent_head(struct perf_cpu_context *ctx, struct perf_event *event)
4093{
4094 struct swevent_hlist *hlist;
4095 u32 event_id = event->attr.config;
4096 u64 type = event->attr.type;
4097
4098 /*
4099 * Event scheduling is always serialized against hlist allocation
4100 * and release. Which makes the protected version suitable here.
4101 * The context lock guarantees that.
4102 */
4103 hlist = rcu_dereference_protected(ctx->swevent_hlist,
4104 lockdep_is_held(&event->ctx->lock));
4105 if (!hlist)
4106 return NULL;
4107
4108 return __find_swevent_head(hlist, type, event_id);
4082} 4109}
4083 4110
4084static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 4111static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
@@ -4095,7 +4122,7 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4095 4122
4096 rcu_read_lock(); 4123 rcu_read_lock();
4097 4124
4098 head = find_swevent_head(cpuctx, type, event_id); 4125 head = find_swevent_head_rcu(cpuctx, type, event_id);
4099 4126
4100 if (!head) 4127 if (!head)
4101 goto end; 4128 goto end;
@@ -4178,7 +4205,7 @@ static int perf_swevent_enable(struct perf_event *event)
4178 perf_swevent_set_period(event); 4205 perf_swevent_set_period(event);
4179 } 4206 }
4180 4207
4181 head = find_swevent_head(cpuctx, event->attr.type, event->attr.config); 4208 head = find_swevent_head(cpuctx, event);
4182 if (WARN_ON_ONCE(!head)) 4209 if (WARN_ON_ONCE(!head))
4183 return -EINVAL; 4210 return -EINVAL;
4184 4211
@@ -4366,6 +4393,14 @@ static const struct pmu perf_ops_task_clock = {
4366 .read = task_clock_perf_event_read, 4393 .read = task_clock_perf_event_read,
4367}; 4394};
4368 4395
4396/* Deref the hlist from the update side */
4397static inline struct swevent_hlist *
4398swevent_hlist_deref(struct perf_cpu_context *cpuctx)
4399{
4400 return rcu_dereference_protected(cpuctx->swevent_hlist,
4401 lockdep_is_held(&cpuctx->hlist_mutex));
4402}
4403
4369static void swevent_hlist_release_rcu(struct rcu_head *rcu_head) 4404static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
4370{ 4405{
4371 struct swevent_hlist *hlist; 4406 struct swevent_hlist *hlist;
@@ -4376,12 +4411,11 @@ static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
4376 4411
4377static void swevent_hlist_release(struct perf_cpu_context *cpuctx) 4412static void swevent_hlist_release(struct perf_cpu_context *cpuctx)
4378{ 4413{
4379 struct swevent_hlist *hlist; 4414 struct swevent_hlist *hlist = swevent_hlist_deref(cpuctx);
4380 4415
4381 if (!cpuctx->swevent_hlist) 4416 if (!hlist)
4382 return; 4417 return;
4383 4418
4384 hlist = cpuctx->swevent_hlist;
4385 rcu_assign_pointer(cpuctx->swevent_hlist, NULL); 4419 rcu_assign_pointer(cpuctx->swevent_hlist, NULL);
4386 call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu); 4420 call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
4387} 4421}
@@ -4418,7 +4452,7 @@ static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
4418 4452
4419 mutex_lock(&cpuctx->hlist_mutex); 4453 mutex_lock(&cpuctx->hlist_mutex);
4420 4454
4421 if (!cpuctx->swevent_hlist && cpu_online(cpu)) { 4455 if (!swevent_hlist_deref(cpuctx) && cpu_online(cpu)) {
4422 struct swevent_hlist *hlist; 4456 struct swevent_hlist *hlist;
4423 4457
4424 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); 4458 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);