aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Shishkin <alexander.shishkin@linux.intel.com>2015-01-14 07:18:18 -0500
committerIngo Molnar <mingo@kernel.org>2015-04-02 11:14:16 -0400
commit1a5941312414c71dece6717da9a0fa1303127afa (patch)
treed8970d17a084578c546ca615705c780c71b1d33f
parent2023a0d2829e521fe6ad6b9907f3f90bfbf57142 (diff)
perf: Add wakeup watermark control to the AUX area
When AUX area gets a certain amount of new data, we want to wake up userspace to collect it. This adds a new control to specify how much data will cause a wakeup. This is then passed down to pmu drivers via output handle's "wakeup" field, so that the driver can find the nearest point where it can generate an interrupt. We repurpose __reserved_2 in the event attribute for this, even though it was never checked to be zero before, aux_watermark will only matter for new AUX-aware code, so the old code should still be fine. Signed-off-by: Alexander Shishkin <alexander.shishkin@linux.intel.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Borislav Petkov <bp@alien8.de> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Kaixu Xia <kaixu.xia@linaro.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mike Galbraith <efault@gmx.de> Cc: Paul Mackerras <paulus@samba.org> Cc: Robert Richter <rric@kernel.org> Cc: Stephane Eranian <eranian@google.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: acme@infradead.org Cc: adrian.hunter@intel.com Cc: kan.liang@intel.com Cc: markus.t.metzger@intel.com Cc: mathieu.poirier@linaro.org Link: http://lkml.kernel.org/r/1421237903-181015-10-git-send-email-alexander.shishkin@linux.intel.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
-rw-r--r--include/uapi/linux/perf_event.h7
-rw-r--r--kernel/events/core.c3
-rw-r--r--kernel/events/internal.h4
-rw-r--r--kernel/events/ring_buffer.c22
4 files changed, 31 insertions, 5 deletions
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index 29ef2f73bb4a..84819546c8ce 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -261,6 +261,7 @@ enum perf_event_read_format {
261#define PERF_ATTR_SIZE_VER3 96 /* add: sample_regs_user */ 261#define PERF_ATTR_SIZE_VER3 96 /* add: sample_regs_user */
262 /* add: sample_stack_user */ 262 /* add: sample_stack_user */
263#define PERF_ATTR_SIZE_VER4 104 /* add: sample_regs_intr */ 263#define PERF_ATTR_SIZE_VER4 104 /* add: sample_regs_intr */
264#define PERF_ATTR_SIZE_VER5 112 /* add: aux_watermark */
264 265
265/* 266/*
266 * Hardware event_id to monitor via a performance monitoring event: 267 * Hardware event_id to monitor via a performance monitoring event:
@@ -366,6 +367,12 @@ struct perf_event_attr {
366 * See asm/perf_regs.h for details. 367 * See asm/perf_regs.h for details.
367 */ 368 */
368 __u64 sample_regs_intr; 369 __u64 sample_regs_intr;
370
371 /*
372 * Wakeup watermark for AUX area
373 */
374 __u32 aux_watermark;
375 __u32 __reserved_2; /* align to __u64 */
369}; 376};
370 377
371#define perf_flags(attr) (*(&(attr)->read_format + 1)) 378#define perf_flags(attr) (*(&(attr)->read_format + 1))
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 81e8d14ac59a..31f6b504ad62 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -4677,7 +4677,8 @@ accounting:
4677 perf_event_init_userpage(event); 4677 perf_event_init_userpage(event);
4678 perf_event_update_userpage(event); 4678 perf_event_update_userpage(event);
4679 } else { 4679 } else {
4680 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, flags); 4680 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
4681 event->attr.aux_watermark, flags);
4681 if (!ret) 4682 if (!ret)
4682 rb->aux_mmap_locked = extra; 4683 rb->aux_mmap_locked = extra;
4683 } 4684 }
diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index ffd51d9f5945..9f6ce9ba4a04 100644
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -27,6 +27,7 @@ struct ring_buffer {
27 local_t lost; /* nr records lost */ 27 local_t lost; /* nr records lost */
28 28
29 long watermark; /* wakeup watermark */ 29 long watermark; /* wakeup watermark */
30 long aux_watermark;
30 /* poll crap */ 31 /* poll crap */
31 spinlock_t event_lock; 32 spinlock_t event_lock;
32 struct list_head event_list; 33 struct list_head event_list;
@@ -38,6 +39,7 @@ struct ring_buffer {
38 /* AUX area */ 39 /* AUX area */
39 local_t aux_head; 40 local_t aux_head;
40 local_t aux_nest; 41 local_t aux_nest;
42 local_t aux_wakeup;
41 unsigned long aux_pgoff; 43 unsigned long aux_pgoff;
42 int aux_nr_pages; 44 int aux_nr_pages;
43 int aux_overwrite; 45 int aux_overwrite;
@@ -57,7 +59,7 @@ extern struct ring_buffer *
57rb_alloc(int nr_pages, long watermark, int cpu, int flags); 59rb_alloc(int nr_pages, long watermark, int cpu, int flags);
58extern void perf_event_wakeup(struct perf_event *event); 60extern void perf_event_wakeup(struct perf_event *event);
59extern int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event, 61extern int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event,
60 pgoff_t pgoff, int nr_pages, int flags); 62 pgoff_t pgoff, int nr_pages, long watermark, int flags);
61extern void rb_free_aux(struct ring_buffer *rb); 63extern void rb_free_aux(struct ring_buffer *rb);
62extern struct ring_buffer *ring_buffer_get(struct perf_event *event); 64extern struct ring_buffer *ring_buffer_get(struct perf_event *event);
63extern void ring_buffer_put(struct ring_buffer *rb); 65extern void ring_buffer_put(struct ring_buffer *rb);
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index 67b328337a41..232f00f273cb 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -296,6 +296,7 @@ void *perf_aux_output_begin(struct perf_output_handle *handle,
296 */ 296 */
297 if (!rb->aux_overwrite) { 297 if (!rb->aux_overwrite) {
298 aux_tail = ACCESS_ONCE(rb->user_page->aux_tail); 298 aux_tail = ACCESS_ONCE(rb->user_page->aux_tail);
299 handle->wakeup = local_read(&rb->aux_wakeup) + rb->aux_watermark;
299 if (aux_head - aux_tail < perf_aux_size(rb)) 300 if (aux_head - aux_tail < perf_aux_size(rb))
300 handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb)); 301 handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));
301 302
@@ -359,9 +360,12 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size,
359 perf_event_aux_event(handle->event, aux_head, size, flags); 360 perf_event_aux_event(handle->event, aux_head, size, flags);
360 } 361 }
361 362
362 rb->user_page->aux_head = local_read(&rb->aux_head); 363 aux_head = rb->user_page->aux_head = local_read(&rb->aux_head);
363 364
364 perf_output_wakeup(handle); 365 if (aux_head - local_read(&rb->aux_wakeup) >= rb->aux_watermark) {
366 perf_output_wakeup(handle);
367 local_add(rb->aux_watermark, &rb->aux_wakeup);
368 }
365 handle->event = NULL; 369 handle->event = NULL;
366 370
367 local_set(&rb->aux_nest, 0); 371 local_set(&rb->aux_nest, 0);
@@ -383,6 +387,14 @@ int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)
383 387
384 local_add(size, &rb->aux_head); 388 local_add(size, &rb->aux_head);
385 389
390 aux_head = rb->user_page->aux_head = local_read(&rb->aux_head);
391 if (aux_head - local_read(&rb->aux_wakeup) >= rb->aux_watermark) {
392 perf_output_wakeup(handle);
393 local_add(rb->aux_watermark, &rb->aux_wakeup);
394 handle->wakeup = local_read(&rb->aux_wakeup) +
395 rb->aux_watermark;
396 }
397
386 handle->head = aux_head; 398 handle->head = aux_head;
387 handle->size -= size; 399 handle->size -= size;
388 400
@@ -433,7 +445,7 @@ static void rb_free_aux_page(struct ring_buffer *rb, int idx)
433} 445}
434 446
435int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event, 447int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event,
436 pgoff_t pgoff, int nr_pages, int flags) 448 pgoff_t pgoff, int nr_pages, long watermark, int flags)
437{ 449{
438 bool overwrite = !(flags & RING_BUFFER_WRITABLE); 450 bool overwrite = !(flags & RING_BUFFER_WRITABLE);
439 int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); 451 int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu);
@@ -497,6 +509,10 @@ int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event,
497 atomic_set(&rb->aux_refcount, 1); 509 atomic_set(&rb->aux_refcount, 1);
498 510
499 rb->aux_overwrite = overwrite; 511 rb->aux_overwrite = overwrite;
512 rb->aux_watermark = watermark;
513
514 if (!rb->aux_watermark && !rb->aux_overwrite)
515 rb->aux_watermark = nr_pages << (PAGE_SHIFT - 1);
500 516
501out: 517out:
502 if (!ret) 518 if (!ret)