aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/events
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/events')
-rw-r--r--kernel/events/core.c63
-rw-r--r--kernel/events/internal.h1
-rw-r--r--kernel/events/ring_buffer.c10
3 files changed, 31 insertions, 43 deletions
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 270e32f9fc06..dbd1ca75bd3c 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -3972,7 +3972,7 @@ void perf_prepare_sample(struct perf_event_header *header,
3972 } 3972 }
3973} 3973}
3974 3974
3975static void perf_event_output(struct perf_event *event, int nmi, 3975static void perf_event_output(struct perf_event *event,
3976 struct perf_sample_data *data, 3976 struct perf_sample_data *data,
3977 struct pt_regs *regs) 3977 struct pt_regs *regs)
3978{ 3978{
@@ -3984,7 +3984,7 @@ static void perf_event_output(struct perf_event *event, int nmi,
3984 3984
3985 perf_prepare_sample(&header, data, event, regs); 3985 perf_prepare_sample(&header, data, event, regs);
3986 3986
3987 if (perf_output_begin(&handle, event, header.size, nmi, 1)) 3987 if (perf_output_begin(&handle, event, header.size, 1))
3988 goto exit; 3988 goto exit;
3989 3989
3990 perf_output_sample(&handle, &header, data, event); 3990 perf_output_sample(&handle, &header, data, event);
@@ -4024,7 +4024,7 @@ perf_event_read_event(struct perf_event *event,
4024 int ret; 4024 int ret;
4025 4025
4026 perf_event_header__init_id(&read_event.header, &sample, event); 4026 perf_event_header__init_id(&read_event.header, &sample, event);
4027 ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0); 4027 ret = perf_output_begin(&handle, event, read_event.header.size, 0);
4028 if (ret) 4028 if (ret)
4029 return; 4029 return;
4030 4030
@@ -4067,7 +4067,7 @@ static void perf_event_task_output(struct perf_event *event,
4067 perf_event_header__init_id(&task_event->event_id.header, &sample, event); 4067 perf_event_header__init_id(&task_event->event_id.header, &sample, event);
4068 4068
4069 ret = perf_output_begin(&handle, event, 4069 ret = perf_output_begin(&handle, event,
4070 task_event->event_id.header.size, 0, 0); 4070 task_event->event_id.header.size, 0);
4071 if (ret) 4071 if (ret)
4072 goto out; 4072 goto out;
4073 4073
@@ -4204,7 +4204,7 @@ static void perf_event_comm_output(struct perf_event *event,
4204 4204
4205 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); 4205 perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4206 ret = perf_output_begin(&handle, event, 4206 ret = perf_output_begin(&handle, event,
4207 comm_event->event_id.header.size, 0, 0); 4207 comm_event->event_id.header.size, 0);
4208 4208
4209 if (ret) 4209 if (ret)
4210 goto out; 4210 goto out;
@@ -4351,7 +4351,7 @@ static void perf_event_mmap_output(struct perf_event *event,
4351 4351
4352 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); 4352 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4353 ret = perf_output_begin(&handle, event, 4353 ret = perf_output_begin(&handle, event,
4354 mmap_event->event_id.header.size, 0, 0); 4354 mmap_event->event_id.header.size, 0);
4355 if (ret) 4355 if (ret)
4356 goto out; 4356 goto out;
4357 4357
@@ -4546,7 +4546,7 @@ static void perf_log_throttle(struct perf_event *event, int enable)
4546 perf_event_header__init_id(&throttle_event.header, &sample, event); 4546 perf_event_header__init_id(&throttle_event.header, &sample, event);
4547 4547
4548 ret = perf_output_begin(&handle, event, 4548 ret = perf_output_begin(&handle, event,
4549 throttle_event.header.size, 1, 0); 4549 throttle_event.header.size, 0);
4550 if (ret) 4550 if (ret)
4551 return; 4551 return;
4552 4552
@@ -4559,7 +4559,7 @@ static void perf_log_throttle(struct perf_event *event, int enable)
4559 * Generic event overflow handling, sampling. 4559 * Generic event overflow handling, sampling.
4560 */ 4560 */
4561 4561
4562static int __perf_event_overflow(struct perf_event *event, int nmi, 4562static int __perf_event_overflow(struct perf_event *event,
4563 int throttle, struct perf_sample_data *data, 4563 int throttle, struct perf_sample_data *data,
4564 struct pt_regs *regs) 4564 struct pt_regs *regs)
4565{ 4565{
@@ -4602,34 +4602,28 @@ static int __perf_event_overflow(struct perf_event *event, int nmi,
4602 if (events && atomic_dec_and_test(&event->event_limit)) { 4602 if (events && atomic_dec_and_test(&event->event_limit)) {
4603 ret = 1; 4603 ret = 1;
4604 event->pending_kill = POLL_HUP; 4604 event->pending_kill = POLL_HUP;
4605 if (nmi) { 4605 event->pending_disable = 1;
4606 event->pending_disable = 1; 4606 irq_work_queue(&event->pending);
4607 irq_work_queue(&event->pending);
4608 } else
4609 perf_event_disable(event);
4610 } 4607 }
4611 4608
4612 if (event->overflow_handler) 4609 if (event->overflow_handler)
4613 event->overflow_handler(event, nmi, data, regs); 4610 event->overflow_handler(event, data, regs);
4614 else 4611 else
4615 perf_event_output(event, nmi, data, regs); 4612 perf_event_output(event, data, regs);
4616 4613
4617 if (event->fasync && event->pending_kill) { 4614 if (event->fasync && event->pending_kill) {
4618 if (nmi) { 4615 event->pending_wakeup = 1;
4619 event->pending_wakeup = 1; 4616 irq_work_queue(&event->pending);
4620 irq_work_queue(&event->pending);
4621 } else
4622 perf_event_wakeup(event);
4623 } 4617 }
4624 4618
4625 return ret; 4619 return ret;
4626} 4620}
4627 4621
4628int perf_event_overflow(struct perf_event *event, int nmi, 4622int perf_event_overflow(struct perf_event *event,
4629 struct perf_sample_data *data, 4623 struct perf_sample_data *data,
4630 struct pt_regs *regs) 4624 struct pt_regs *regs)
4631{ 4625{
4632 return __perf_event_overflow(event, nmi, 1, data, regs); 4626 return __perf_event_overflow(event, 1, data, regs);
4633} 4627}
4634 4628
4635/* 4629/*
@@ -4678,7 +4672,7 @@ again:
4678} 4672}
4679 4673
4680static void perf_swevent_overflow(struct perf_event *event, u64 overflow, 4674static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4681 int nmi, struct perf_sample_data *data, 4675 struct perf_sample_data *data,
4682 struct pt_regs *regs) 4676 struct pt_regs *regs)
4683{ 4677{
4684 struct hw_perf_event *hwc = &event->hw; 4678 struct hw_perf_event *hwc = &event->hw;
@@ -4692,7 +4686,7 @@ static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4692 return; 4686 return;
4693 4687
4694 for (; overflow; overflow--) { 4688 for (; overflow; overflow--) {
4695 if (__perf_event_overflow(event, nmi, throttle, 4689 if (__perf_event_overflow(event, throttle,
4696 data, regs)) { 4690 data, regs)) {
4697 /* 4691 /*
4698 * We inhibit the overflow from happening when 4692 * We inhibit the overflow from happening when
@@ -4705,7 +4699,7 @@ static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4705} 4699}
4706 4700
4707static void perf_swevent_event(struct perf_event *event, u64 nr, 4701static void perf_swevent_event(struct perf_event *event, u64 nr,
4708 int nmi, struct perf_sample_data *data, 4702 struct perf_sample_data *data,
4709 struct pt_regs *regs) 4703 struct pt_regs *regs)
4710{ 4704{
4711 struct hw_perf_event *hwc = &event->hw; 4705 struct hw_perf_event *hwc = &event->hw;
@@ -4719,12 +4713,12 @@ static void perf_swevent_event(struct perf_event *event, u64 nr,
4719 return; 4713 return;
4720 4714
4721 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) 4715 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4722 return perf_swevent_overflow(event, 1, nmi, data, regs); 4716 return perf_swevent_overflow(event, 1, data, regs);
4723 4717
4724 if (local64_add_negative(nr, &hwc->period_left)) 4718 if (local64_add_negative(nr, &hwc->period_left))
4725 return; 4719 return;
4726 4720
4727 perf_swevent_overflow(event, 0, nmi, data, regs); 4721 perf_swevent_overflow(event, 0, data, regs);
4728} 4722}
4729 4723
4730static int perf_exclude_event(struct perf_event *event, 4724static int perf_exclude_event(struct perf_event *event,
@@ -4812,7 +4806,7 @@ find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
4812} 4806}
4813 4807
4814static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 4808static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4815 u64 nr, int nmi, 4809 u64 nr,
4816 struct perf_sample_data *data, 4810 struct perf_sample_data *data,
4817 struct pt_regs *regs) 4811 struct pt_regs *regs)
4818{ 4812{
@@ -4828,7 +4822,7 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4828 4822
4829 hlist_for_each_entry_rcu(event, node, head, hlist_entry) { 4823 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4830 if (perf_swevent_match(event, type, event_id, data, regs)) 4824 if (perf_swevent_match(event, type, event_id, data, regs))
4831 perf_swevent_event(event, nr, nmi, data, regs); 4825 perf_swevent_event(event, nr, data, regs);
4832 } 4826 }
4833end: 4827end:
4834 rcu_read_unlock(); 4828 rcu_read_unlock();
@@ -4849,8 +4843,7 @@ inline void perf_swevent_put_recursion_context(int rctx)
4849 put_recursion_context(swhash->recursion, rctx); 4843 put_recursion_context(swhash->recursion, rctx);
4850} 4844}
4851 4845
4852void __perf_sw_event(u32 event_id, u64 nr, int nmi, 4846void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
4853 struct pt_regs *regs, u64 addr)
4854{ 4847{
4855 struct perf_sample_data data; 4848 struct perf_sample_data data;
4856 int rctx; 4849 int rctx;
@@ -4862,7 +4855,7 @@ void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4862 4855
4863 perf_sample_data_init(&data, addr); 4856 perf_sample_data_init(&data, addr);
4864 4857
4865 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs); 4858 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
4866 4859
4867 perf_swevent_put_recursion_context(rctx); 4860 perf_swevent_put_recursion_context(rctx);
4868 preempt_enable_notrace(); 4861 preempt_enable_notrace();
@@ -5110,7 +5103,7 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5110 5103
5111 hlist_for_each_entry_rcu(event, node, head, hlist_entry) { 5104 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5112 if (perf_tp_event_match(event, &data, regs)) 5105 if (perf_tp_event_match(event, &data, regs))
5113 perf_swevent_event(event, count, 1, &data, regs); 5106 perf_swevent_event(event, count, &data, regs);
5114 } 5107 }
5115 5108
5116 perf_swevent_put_recursion_context(rctx); 5109 perf_swevent_put_recursion_context(rctx);
@@ -5203,7 +5196,7 @@ void perf_bp_event(struct perf_event *bp, void *data)
5203 perf_sample_data_init(&sample, bp->attr.bp_addr); 5196 perf_sample_data_init(&sample, bp->attr.bp_addr);
5204 5197
5205 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 5198 if (!bp->hw.state && !perf_exclude_event(bp, regs))
5206 perf_swevent_event(bp, 1, 1, &sample, regs); 5199 perf_swevent_event(bp, 1, &sample, regs);
5207} 5200}
5208#endif 5201#endif
5209 5202
@@ -5232,7 +5225,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5232 5225
5233 if (regs && !perf_exclude_event(event, regs)) { 5226 if (regs && !perf_exclude_event(event, regs)) {
5234 if (!(event->attr.exclude_idle && current->pid == 0)) 5227 if (!(event->attr.exclude_idle && current->pid == 0))
5235 if (perf_event_overflow(event, 0, &data, regs)) 5228 if (perf_event_overflow(event, &data, regs))
5236 ret = HRTIMER_NORESTART; 5229 ret = HRTIMER_NORESTART;
5237 } 5230 }
5238 5231
diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index 114f27f3a624..09097dd8116c 100644
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -27,7 +27,6 @@ struct ring_buffer {
27 void *data_pages[0]; 27 void *data_pages[0];
28}; 28};
29 29
30
31extern void rb_free(struct ring_buffer *rb); 30extern void rb_free(struct ring_buffer *rb);
32extern struct ring_buffer * 31extern struct ring_buffer *
33rb_alloc(int nr_pages, long watermark, int cpu, int flags); 32rb_alloc(int nr_pages, long watermark, int cpu, int flags);
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index fc2701c99207..8b3b73630fa4 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -38,11 +38,8 @@ static void perf_output_wakeup(struct perf_output_handle *handle)
38{ 38{
39 atomic_set(&handle->rb->poll, POLL_IN); 39 atomic_set(&handle->rb->poll, POLL_IN);
40 40
41 if (handle->nmi) { 41 handle->event->pending_wakeup = 1;
42 handle->event->pending_wakeup = 1; 42 irq_work_queue(&handle->event->pending);
43 irq_work_queue(&handle->event->pending);
44 } else
45 perf_event_wakeup(handle->event);
46} 43}
47 44
48/* 45/*
@@ -102,7 +99,7 @@ out:
102 99
103int perf_output_begin(struct perf_output_handle *handle, 100int perf_output_begin(struct perf_output_handle *handle,
104 struct perf_event *event, unsigned int size, 101 struct perf_event *event, unsigned int size,
105 int nmi, int sample) 102 int sample)
106{ 103{
107 struct ring_buffer *rb; 104 struct ring_buffer *rb;
108 unsigned long tail, offset, head; 105 unsigned long tail, offset, head;
@@ -127,7 +124,6 @@ int perf_output_begin(struct perf_output_handle *handle,
127 124
128 handle->rb = rb; 125 handle->rb = rb;
129 handle->event = event; 126 handle->event = event;
130 handle->nmi = nmi;
131 handle->sample = sample; 127 handle->sample = sample;
132 128
133 if (!rb->nr_pages) 129 if (!rb->nr_pages)