ring-buffer: Add percentage of ring buffer full to wake up reader

Instead of just waiting for a page to be full before waking up a pending
reader, allow the reader to pass in a "percentage" of pages that have
content before waking up a reader. This should help keep the process of
reading the events not cause wake ups that constantly cause reading of the
buffer.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>

1 files changed, 66 insertions, 5 deletions

diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 65bd4616220d..9edb628603ab 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -487,6 +487,9 @@ struct ring_buffer_per_cpu {
         local_t                         dropped_events;
         local_t                         committing;
         local_t                         commits;
+        local_t                         pages_touched;
+        local_t                         pages_read;
+        size_t                          shortest_full;
         unsigned long                   read;
         unsigned long                   read_bytes;
         u64                             write_stamp;
@@ -529,6 +532,41 @@ struct ring_buffer_iter {
         u64                             read_stamp;
 };
 
+/**
+ * ring_buffer_nr_pages - get the number of buffer pages in the ring buffer
+ * @buffer: The ring_buffer to get the number of pages from
+ * @cpu: The cpu of the ring_buffer to get the number of pages from
+ *
+ * Returns the number of pages used by a per_cpu buffer of the ring buffer.
+ */
+size_t ring_buffer_nr_pages(struct ring_buffer *buffer, int cpu)
+{
+        return buffer->buffers[cpu]->nr_pages;
+}
+
+/**
+ * ring_buffer_nr_pages_dirty - get the number of used pages in the ring buffer
+ * @buffer: The ring_buffer to get the number of pages from
+ * @cpu: The cpu of the ring_buffer to get the number of pages from
+ *
+ * Returns the number of pages that have content in the ring buffer.
+ */
+size_t ring_buffer_nr_dirty_pages(struct ring_buffer *buffer, int cpu)
+{
+        size_t read;
+        size_t cnt;
+
+        read = local_read(&buffer->buffers[cpu]->pages_read);
+        cnt = local_read(&buffer->buffers[cpu]->pages_touched);
+        /* The reader can read an empty page, but not more than that */
+        if (cnt < read) {
+                WARN_ON_ONCE(read > cnt + 1);
+                return 0;
+        }
+
+        return cnt - read;
+}
+
 /*
  * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
  *
@@ -556,7 +594,7 @@ static void rb_wake_up_waiters(struct irq_work *work)
  * as data is added to any of the @buffer's cpu buffers. Otherwise
  * it will wait for data to be added to a specific cpu buffer.
  */
-int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
+int ring_buffer_wait(struct ring_buffer *buffer, int cpu, int full)
 {
         struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer);
         DEFINE_WAIT(wait);
@@ -571,7 +609,7 @@ int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
         if (cpu == RING_BUFFER_ALL_CPUS) {
                 work = &buffer->irq_work;
                 /* Full only makes sense on per cpu reads */
-                full = false;
+                full = 0;
         } else {
                 if (!cpumask_test_cpu(cpu, buffer->cpumask))
                         return -ENODEV;
@@ -623,15 +661,22 @@ int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
                     !ring_buffer_empty_cpu(buffer, cpu)) {
                         unsigned long flags;
                         bool pagebusy;
+                        size_t nr_pages;
+                        size_t dirty;
 
                         if (!full)
                                 break;
 
                         raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
                         pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
+                        nr_pages = cpu_buffer->nr_pages;
+                        dirty = ring_buffer_nr_dirty_pages(buffer, cpu);
+                        if (!cpu_buffer->shortest_full ||
+                            cpu_buffer->shortest_full < full)
+                                cpu_buffer->shortest_full = full;
                         raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
-
-                        if (!pagebusy)
+                        if (!pagebusy &&
+                            (!nr_pages || (dirty * 100) > full * nr_pages))
                                 break;
                 }
 
@@ -1054,6 +1099,7 @@ static void rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer,
         old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
         old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);
 
+        local_inc(&cpu_buffer->pages_touched);
         /*
          * Just make sure we have seen our old_write and synchronize
          * with any interrupts that come in.
@@ -2603,6 +2649,16 @@ rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)
         pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
 
         if (!pagebusy && cpu_buffer->irq_work.full_waiters_pending) {
+                size_t nr_pages;
+                size_t dirty;
+                size_t full;
+
+                full = cpu_buffer->shortest_full;
+                nr_pages = cpu_buffer->nr_pages;
+                dirty = ring_buffer_nr_dirty_pages(buffer, cpu_buffer->cpu);
+                if (full && nr_pages && (dirty * 100) <= full * nr_pages)
+                        return;
+
                 cpu_buffer->irq_work.wakeup_full = true;
                 cpu_buffer->irq_work.full_waiters_pending = false;
                 /* irq_work_queue() supplies it's own memory barriers */
@@ -3732,13 +3788,15 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
                 goto spin;
 
         /*
-         * Yeah! We succeeded in replacing the page.
+         * Yay! We succeeded in replacing the page.
          *
          * Now make the new head point back to the reader page.
          */
         rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
         rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
 
+        local_inc(&cpu_buffer->pages_read);
+
         /* Finally update the reader page to the new head */
         cpu_buffer->reader_page = reader;
         cpu_buffer->reader_page->read = 0;
@@ -4334,6 +4392,9 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
         local_set(&cpu_buffer->entries, 0);
         local_set(&cpu_buffer->committing, 0);
         local_set(&cpu_buffer->commits, 0);
+        local_set(&cpu_buffer->pages_touched, 0);
+        local_set(&cpu_buffer->pages_read, 0);
+        cpu_buffer->shortest_full = 0;
         cpu_buffer->read = 0;
         cpu_buffer->read_bytes = 0;