diff options
Diffstat (limited to 'kernel/events')
-rw-r--r-- | kernel/events/core.c | 63 | ||||
-rw-r--r-- | kernel/events/internal.h | 1 | ||||
-rw-r--r-- | kernel/events/ring_buffer.c | 10 |
3 files changed, 31 insertions, 43 deletions
diff --git a/kernel/events/core.c b/kernel/events/core.c index 270e32f9fc06..dbd1ca75bd3c 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c | |||
@@ -3972,7 +3972,7 @@ void perf_prepare_sample(struct perf_event_header *header, | |||
3972 | } | 3972 | } |
3973 | } | 3973 | } |
3974 | 3974 | ||
3975 | static void perf_event_output(struct perf_event *event, int nmi, | 3975 | static void perf_event_output(struct perf_event *event, |
3976 | struct perf_sample_data *data, | 3976 | struct perf_sample_data *data, |
3977 | struct pt_regs *regs) | 3977 | struct pt_regs *regs) |
3978 | { | 3978 | { |
@@ -3984,7 +3984,7 @@ static void perf_event_output(struct perf_event *event, int nmi, | |||
3984 | 3984 | ||
3985 | perf_prepare_sample(&header, data, event, regs); | 3985 | perf_prepare_sample(&header, data, event, regs); |
3986 | 3986 | ||
3987 | if (perf_output_begin(&handle, event, header.size, nmi, 1)) | 3987 | if (perf_output_begin(&handle, event, header.size, 1)) |
3988 | goto exit; | 3988 | goto exit; |
3989 | 3989 | ||
3990 | perf_output_sample(&handle, &header, data, event); | 3990 | perf_output_sample(&handle, &header, data, event); |
@@ -4024,7 +4024,7 @@ perf_event_read_event(struct perf_event *event, | |||
4024 | int ret; | 4024 | int ret; |
4025 | 4025 | ||
4026 | perf_event_header__init_id(&read_event.header, &sample, event); | 4026 | perf_event_header__init_id(&read_event.header, &sample, event); |
4027 | ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0); | 4027 | ret = perf_output_begin(&handle, event, read_event.header.size, 0); |
4028 | if (ret) | 4028 | if (ret) |
4029 | return; | 4029 | return; |
4030 | 4030 | ||
@@ -4067,7 +4067,7 @@ static void perf_event_task_output(struct perf_event *event, | |||
4067 | perf_event_header__init_id(&task_event->event_id.header, &sample, event); | 4067 | perf_event_header__init_id(&task_event->event_id.header, &sample, event); |
4068 | 4068 | ||
4069 | ret = perf_output_begin(&handle, event, | 4069 | ret = perf_output_begin(&handle, event, |
4070 | task_event->event_id.header.size, 0, 0); | 4070 | task_event->event_id.header.size, 0); |
4071 | if (ret) | 4071 | if (ret) |
4072 | goto out; | 4072 | goto out; |
4073 | 4073 | ||
@@ -4204,7 +4204,7 @@ static void perf_event_comm_output(struct perf_event *event, | |||
4204 | 4204 | ||
4205 | perf_event_header__init_id(&comm_event->event_id.header, &sample, event); | 4205 | perf_event_header__init_id(&comm_event->event_id.header, &sample, event); |
4206 | ret = perf_output_begin(&handle, event, | 4206 | ret = perf_output_begin(&handle, event, |
4207 | comm_event->event_id.header.size, 0, 0); | 4207 | comm_event->event_id.header.size, 0); |
4208 | 4208 | ||
4209 | if (ret) | 4209 | if (ret) |
4210 | goto out; | 4210 | goto out; |
@@ -4351,7 +4351,7 @@ static void perf_event_mmap_output(struct perf_event *event, | |||
4351 | 4351 | ||
4352 | perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); | 4352 | perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); |
4353 | ret = perf_output_begin(&handle, event, | 4353 | ret = perf_output_begin(&handle, event, |
4354 | mmap_event->event_id.header.size, 0, 0); | 4354 | mmap_event->event_id.header.size, 0); |
4355 | if (ret) | 4355 | if (ret) |
4356 | goto out; | 4356 | goto out; |
4357 | 4357 | ||
@@ -4546,7 +4546,7 @@ static void perf_log_throttle(struct perf_event *event, int enable) | |||
4546 | perf_event_header__init_id(&throttle_event.header, &sample, event); | 4546 | perf_event_header__init_id(&throttle_event.header, &sample, event); |
4547 | 4547 | ||
4548 | ret = perf_output_begin(&handle, event, | 4548 | ret = perf_output_begin(&handle, event, |
4549 | throttle_event.header.size, 1, 0); | 4549 | throttle_event.header.size, 0); |
4550 | if (ret) | 4550 | if (ret) |
4551 | return; | 4551 | return; |
4552 | 4552 | ||
@@ -4559,7 +4559,7 @@ static void perf_log_throttle(struct perf_event *event, int enable) | |||
4559 | * Generic event overflow handling, sampling. | 4559 | * Generic event overflow handling, sampling. |
4560 | */ | 4560 | */ |
4561 | 4561 | ||
4562 | static int __perf_event_overflow(struct perf_event *event, int nmi, | 4562 | static int __perf_event_overflow(struct perf_event *event, |
4563 | int throttle, struct perf_sample_data *data, | 4563 | int throttle, struct perf_sample_data *data, |
4564 | struct pt_regs *regs) | 4564 | struct pt_regs *regs) |
4565 | { | 4565 | { |
@@ -4602,34 +4602,28 @@ static int __perf_event_overflow(struct perf_event *event, int nmi, | |||
4602 | if (events && atomic_dec_and_test(&event->event_limit)) { | 4602 | if (events && atomic_dec_and_test(&event->event_limit)) { |
4603 | ret = 1; | 4603 | ret = 1; |
4604 | event->pending_kill = POLL_HUP; | 4604 | event->pending_kill = POLL_HUP; |
4605 | if (nmi) { | 4605 | event->pending_disable = 1; |
4606 | event->pending_disable = 1; | 4606 | irq_work_queue(&event->pending); |
4607 | irq_work_queue(&event->pending); | ||
4608 | } else | ||
4609 | perf_event_disable(event); | ||
4610 | } | 4607 | } |
4611 | 4608 | ||
4612 | if (event->overflow_handler) | 4609 | if (event->overflow_handler) |
4613 | event->overflow_handler(event, nmi, data, regs); | 4610 | event->overflow_handler(event, data, regs); |
4614 | else | 4611 | else |
4615 | perf_event_output(event, nmi, data, regs); | 4612 | perf_event_output(event, data, regs); |
4616 | 4613 | ||
4617 | if (event->fasync && event->pending_kill) { | 4614 | if (event->fasync && event->pending_kill) { |
4618 | if (nmi) { | 4615 | event->pending_wakeup = 1; |
4619 | event->pending_wakeup = 1; | 4616 | irq_work_queue(&event->pending); |
4620 | irq_work_queue(&event->pending); | ||
4621 | } else | ||
4622 | perf_event_wakeup(event); | ||
4623 | } | 4617 | } |
4624 | 4618 | ||
4625 | return ret; | 4619 | return ret; |
4626 | } | 4620 | } |
4627 | 4621 | ||
4628 | int perf_event_overflow(struct perf_event *event, int nmi, | 4622 | int perf_event_overflow(struct perf_event *event, |
4629 | struct perf_sample_data *data, | 4623 | struct perf_sample_data *data, |
4630 | struct pt_regs *regs) | 4624 | struct pt_regs *regs) |
4631 | { | 4625 | { |
4632 | return __perf_event_overflow(event, nmi, 1, data, regs); | 4626 | return __perf_event_overflow(event, 1, data, regs); |
4633 | } | 4627 | } |
4634 | 4628 | ||
4635 | /* | 4629 | /* |
@@ -4678,7 +4672,7 @@ again: | |||
4678 | } | 4672 | } |
4679 | 4673 | ||
4680 | static void perf_swevent_overflow(struct perf_event *event, u64 overflow, | 4674 | static void perf_swevent_overflow(struct perf_event *event, u64 overflow, |
4681 | int nmi, struct perf_sample_data *data, | 4675 | struct perf_sample_data *data, |
4682 | struct pt_regs *regs) | 4676 | struct pt_regs *regs) |
4683 | { | 4677 | { |
4684 | struct hw_perf_event *hwc = &event->hw; | 4678 | struct hw_perf_event *hwc = &event->hw; |
@@ -4692,7 +4686,7 @@ static void perf_swevent_overflow(struct perf_event *event, u64 overflow, | |||
4692 | return; | 4686 | return; |
4693 | 4687 | ||
4694 | for (; overflow; overflow--) { | 4688 | for (; overflow; overflow--) { |
4695 | if (__perf_event_overflow(event, nmi, throttle, | 4689 | if (__perf_event_overflow(event, throttle, |
4696 | data, regs)) { | 4690 | data, regs)) { |
4697 | /* | 4691 | /* |
4698 | * We inhibit the overflow from happening when | 4692 | * We inhibit the overflow from happening when |
@@ -4705,7 +4699,7 @@ static void perf_swevent_overflow(struct perf_event *event, u64 overflow, | |||
4705 | } | 4699 | } |
4706 | 4700 | ||
4707 | static void perf_swevent_event(struct perf_event *event, u64 nr, | 4701 | static void perf_swevent_event(struct perf_event *event, u64 nr, |
4708 | int nmi, struct perf_sample_data *data, | 4702 | struct perf_sample_data *data, |
4709 | struct pt_regs *regs) | 4703 | struct pt_regs *regs) |
4710 | { | 4704 | { |
4711 | struct hw_perf_event *hwc = &event->hw; | 4705 | struct hw_perf_event *hwc = &event->hw; |
@@ -4719,12 +4713,12 @@ static void perf_swevent_event(struct perf_event *event, u64 nr, | |||
4719 | return; | 4713 | return; |
4720 | 4714 | ||
4721 | if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) | 4715 | if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) |
4722 | return perf_swevent_overflow(event, 1, nmi, data, regs); | 4716 | return perf_swevent_overflow(event, 1, data, regs); |
4723 | 4717 | ||
4724 | if (local64_add_negative(nr, &hwc->period_left)) | 4718 | if (local64_add_negative(nr, &hwc->period_left)) |
4725 | return; | 4719 | return; |
4726 | 4720 | ||
4727 | perf_swevent_overflow(event, 0, nmi, data, regs); | 4721 | perf_swevent_overflow(event, 0, data, regs); |
4728 | } | 4722 | } |
4729 | 4723 | ||
4730 | static int perf_exclude_event(struct perf_event *event, | 4724 | static int perf_exclude_event(struct perf_event *event, |
@@ -4812,7 +4806,7 @@ find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) | |||
4812 | } | 4806 | } |
4813 | 4807 | ||
4814 | static void do_perf_sw_event(enum perf_type_id type, u32 event_id, | 4808 | static void do_perf_sw_event(enum perf_type_id type, u32 event_id, |
4815 | u64 nr, int nmi, | 4809 | u64 nr, |
4816 | struct perf_sample_data *data, | 4810 | struct perf_sample_data *data, |
4817 | struct pt_regs *regs) | 4811 | struct pt_regs *regs) |
4818 | { | 4812 | { |
@@ -4828,7 +4822,7 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id, | |||
4828 | 4822 | ||
4829 | hlist_for_each_entry_rcu(event, node, head, hlist_entry) { | 4823 | hlist_for_each_entry_rcu(event, node, head, hlist_entry) { |
4830 | if (perf_swevent_match(event, type, event_id, data, regs)) | 4824 | if (perf_swevent_match(event, type, event_id, data, regs)) |
4831 | perf_swevent_event(event, nr, nmi, data, regs); | 4825 | perf_swevent_event(event, nr, data, regs); |
4832 | } | 4826 | } |
4833 | end: | 4827 | end: |
4834 | rcu_read_unlock(); | 4828 | rcu_read_unlock(); |
@@ -4849,8 +4843,7 @@ inline void perf_swevent_put_recursion_context(int rctx) | |||
4849 | put_recursion_context(swhash->recursion, rctx); | 4843 | put_recursion_context(swhash->recursion, rctx); |
4850 | } | 4844 | } |
4851 | 4845 | ||
4852 | void __perf_sw_event(u32 event_id, u64 nr, int nmi, | 4846 | void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) |
4853 | struct pt_regs *regs, u64 addr) | ||
4854 | { | 4847 | { |
4855 | struct perf_sample_data data; | 4848 | struct perf_sample_data data; |
4856 | int rctx; | 4849 | int rctx; |
@@ -4862,7 +4855,7 @@ void __perf_sw_event(u32 event_id, u64 nr, int nmi, | |||
4862 | 4855 | ||
4863 | perf_sample_data_init(&data, addr); | 4856 | perf_sample_data_init(&data, addr); |
4864 | 4857 | ||
4865 | do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs); | 4858 | do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); |
4866 | 4859 | ||
4867 | perf_swevent_put_recursion_context(rctx); | 4860 | perf_swevent_put_recursion_context(rctx); |
4868 | preempt_enable_notrace(); | 4861 | preempt_enable_notrace(); |
@@ -5110,7 +5103,7 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, | |||
5110 | 5103 | ||
5111 | hlist_for_each_entry_rcu(event, node, head, hlist_entry) { | 5104 | hlist_for_each_entry_rcu(event, node, head, hlist_entry) { |
5112 | if (perf_tp_event_match(event, &data, regs)) | 5105 | if (perf_tp_event_match(event, &data, regs)) |
5113 | perf_swevent_event(event, count, 1, &data, regs); | 5106 | perf_swevent_event(event, count, &data, regs); |
5114 | } | 5107 | } |
5115 | 5108 | ||
5116 | perf_swevent_put_recursion_context(rctx); | 5109 | perf_swevent_put_recursion_context(rctx); |
@@ -5203,7 +5196,7 @@ void perf_bp_event(struct perf_event *bp, void *data) | |||
5203 | perf_sample_data_init(&sample, bp->attr.bp_addr); | 5196 | perf_sample_data_init(&sample, bp->attr.bp_addr); |
5204 | 5197 | ||
5205 | if (!bp->hw.state && !perf_exclude_event(bp, regs)) | 5198 | if (!bp->hw.state && !perf_exclude_event(bp, regs)) |
5206 | perf_swevent_event(bp, 1, 1, &sample, regs); | 5199 | perf_swevent_event(bp, 1, &sample, regs); |
5207 | } | 5200 | } |
5208 | #endif | 5201 | #endif |
5209 | 5202 | ||
@@ -5232,7 +5225,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) | |||
5232 | 5225 | ||
5233 | if (regs && !perf_exclude_event(event, regs)) { | 5226 | if (regs && !perf_exclude_event(event, regs)) { |
5234 | if (!(event->attr.exclude_idle && current->pid == 0)) | 5227 | if (!(event->attr.exclude_idle && current->pid == 0)) |
5235 | if (perf_event_overflow(event, 0, &data, regs)) | 5228 | if (perf_event_overflow(event, &data, regs)) |
5236 | ret = HRTIMER_NORESTART; | 5229 | ret = HRTIMER_NORESTART; |
5237 | } | 5230 | } |
5238 | 5231 | ||
diff --git a/kernel/events/internal.h b/kernel/events/internal.h index 114f27f3a624..09097dd8116c 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h | |||
@@ -27,7 +27,6 @@ struct ring_buffer { | |||
27 | void *data_pages[0]; | 27 | void *data_pages[0]; |
28 | }; | 28 | }; |
29 | 29 | ||
30 | |||
31 | extern void rb_free(struct ring_buffer *rb); | 30 | extern void rb_free(struct ring_buffer *rb); |
32 | extern struct ring_buffer * | 31 | extern struct ring_buffer * |
33 | rb_alloc(int nr_pages, long watermark, int cpu, int flags); | 32 | rb_alloc(int nr_pages, long watermark, int cpu, int flags); |
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index fc2701c99207..8b3b73630fa4 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c | |||
@@ -38,11 +38,8 @@ static void perf_output_wakeup(struct perf_output_handle *handle) | |||
38 | { | 38 | { |
39 | atomic_set(&handle->rb->poll, POLL_IN); | 39 | atomic_set(&handle->rb->poll, POLL_IN); |
40 | 40 | ||
41 | if (handle->nmi) { | 41 | handle->event->pending_wakeup = 1; |
42 | handle->event->pending_wakeup = 1; | 42 | irq_work_queue(&handle->event->pending); |
43 | irq_work_queue(&handle->event->pending); | ||
44 | } else | ||
45 | perf_event_wakeup(handle->event); | ||
46 | } | 43 | } |
47 | 44 | ||
48 | /* | 45 | /* |
@@ -102,7 +99,7 @@ out: | |||
102 | 99 | ||
103 | int perf_output_begin(struct perf_output_handle *handle, | 100 | int perf_output_begin(struct perf_output_handle *handle, |
104 | struct perf_event *event, unsigned int size, | 101 | struct perf_event *event, unsigned int size, |
105 | int nmi, int sample) | 102 | int sample) |
106 | { | 103 | { |
107 | struct ring_buffer *rb; | 104 | struct ring_buffer *rb; |
108 | unsigned long tail, offset, head; | 105 | unsigned long tail, offset, head; |
@@ -127,7 +124,6 @@ int perf_output_begin(struct perf_output_handle *handle, | |||
127 | 124 | ||
128 | handle->rb = rb; | 125 | handle->rb = rb; |
129 | handle->event = event; | 126 | handle->event = event; |
130 | handle->nmi = nmi; | ||
131 | handle->sample = sample; | 127 | handle->sample = sample; |
132 | 128 | ||
133 | if (!rb->nr_pages) | 129 | if (!rb->nr_pages) |