aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/perf_event.c58
1 files changed, 46 insertions, 12 deletions
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index a4fa381db3c2..511677bc1c6a 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -4066,19 +4066,46 @@ static inline u64 swevent_hash(u64 type, u32 event_id)
4066 return hash_64(val, SWEVENT_HLIST_BITS); 4066 return hash_64(val, SWEVENT_HLIST_BITS);
4067} 4067}
4068 4068
4069static struct hlist_head * 4069static inline struct hlist_head *
4070find_swevent_head(struct perf_cpu_context *ctx, u64 type, u32 event_id) 4070__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4071{ 4071{
4072 u64 hash; 4072 u64 hash = swevent_hash(type, event_id);
4073 struct swevent_hlist *hlist; 4073
4074 return &hlist->heads[hash];
4075}
4074 4076
4075 hash = swevent_hash(type, event_id); 4077/* For the read side: events when they trigger */
4078static inline struct hlist_head *
4079find_swevent_head_rcu(struct perf_cpu_context *ctx, u64 type, u32 event_id)
4080{
4081 struct swevent_hlist *hlist;
4076 4082
4077 hlist = rcu_dereference(ctx->swevent_hlist); 4083 hlist = rcu_dereference(ctx->swevent_hlist);
4078 if (!hlist) 4084 if (!hlist)
4079 return NULL; 4085 return NULL;
4080 4086
4081 return &hlist->heads[hash]; 4087 return __find_swevent_head(hlist, type, event_id);
4088}
4089
4090/* For the event head insertion and removal in the hlist */
4091static inline struct hlist_head *
4092find_swevent_head(struct perf_cpu_context *ctx, struct perf_event *event)
4093{
4094 struct swevent_hlist *hlist;
4095 u32 event_id = event->attr.config;
4096 u64 type = event->attr.type;
4097
4098 /*
4099 * Event scheduling is always serialized against hlist allocation
4100 * and release. Which makes the protected version suitable here.
4101 * The context lock guarantees that.
4102 */
4103 hlist = rcu_dereference_protected(ctx->swevent_hlist,
4104 lockdep_is_held(&event->ctx->lock));
4105 if (!hlist)
4106 return NULL;
4107
4108 return __find_swevent_head(hlist, type, event_id);
4082} 4109}
4083 4110
4084static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 4111static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
@@ -4095,7 +4122,7 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4095 4122
4096 rcu_read_lock(); 4123 rcu_read_lock();
4097 4124
4098 head = find_swevent_head(cpuctx, type, event_id); 4125 head = find_swevent_head_rcu(cpuctx, type, event_id);
4099 4126
4100 if (!head) 4127 if (!head)
4101 goto end; 4128 goto end;
@@ -4178,7 +4205,7 @@ static int perf_swevent_enable(struct perf_event *event)
4178 perf_swevent_set_period(event); 4205 perf_swevent_set_period(event);
4179 } 4206 }
4180 4207
4181 head = find_swevent_head(cpuctx, event->attr.type, event->attr.config); 4208 head = find_swevent_head(cpuctx, event);
4182 if (WARN_ON_ONCE(!head)) 4209 if (WARN_ON_ONCE(!head))
4183 return -EINVAL; 4210 return -EINVAL;
4184 4211
@@ -4366,6 +4393,14 @@ static const struct pmu perf_ops_task_clock = {
4366 .read = task_clock_perf_event_read, 4393 .read = task_clock_perf_event_read,
4367}; 4394};
4368 4395
4396/* Deref the hlist from the update side */
4397static inline struct swevent_hlist *
4398swevent_hlist_deref(struct perf_cpu_context *cpuctx)
4399{
4400 return rcu_dereference_protected(cpuctx->swevent_hlist,
4401 lockdep_is_held(&cpuctx->hlist_mutex));
4402}
4403
4369static void swevent_hlist_release_rcu(struct rcu_head *rcu_head) 4404static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
4370{ 4405{
4371 struct swevent_hlist *hlist; 4406 struct swevent_hlist *hlist;
@@ -4376,12 +4411,11 @@ static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
4376 4411
4377static void swevent_hlist_release(struct perf_cpu_context *cpuctx) 4412static void swevent_hlist_release(struct perf_cpu_context *cpuctx)
4378{ 4413{
4379 struct swevent_hlist *hlist; 4414 struct swevent_hlist *hlist = swevent_hlist_deref(cpuctx);
4380 4415
4381 if (!cpuctx->swevent_hlist) 4416 if (!hlist)
4382 return; 4417 return;
4383 4418
4384 hlist = cpuctx->swevent_hlist;
4385 rcu_assign_pointer(cpuctx->swevent_hlist, NULL); 4419 rcu_assign_pointer(cpuctx->swevent_hlist, NULL);
4386 call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu); 4420 call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
4387} 4421}
@@ -4418,7 +4452,7 @@ static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
4418 4452
4419 mutex_lock(&cpuctx->hlist_mutex); 4453 mutex_lock(&cpuctx->hlist_mutex);
4420 4454
4421 if (!cpuctx->swevent_hlist && cpu_online(cpu)) { 4455 if (!swevent_hlist_deref(cpuctx) && cpu_online(cpu)) {
4422 struct swevent_hlist *hlist; 4456 struct swevent_hlist *hlist;
4423 4457
4424 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); 4458 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);