1 files changed, 83 insertions, 45 deletions

diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 925f629658d6..a56e07c8d15b 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -538,16 +538,18 @@ static void rb_wake_up_waiters(struct irq_work *work)
  * ring_buffer_wait - wait for input to the ring buffer
  * @buffer: buffer to wait on
  * @cpu: the cpu buffer to wait on
+ * @full: wait until a full page is available, if @cpu != RING_BUFFER_ALL_CPUS
  *
  * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
  * as data is added to any of the @buffer's cpu buffers. Otherwise
  * it will wait for data to be added to a specific cpu buffer.
  */
-int ring_buffer_wait(struct ring_buffer *buffer, int cpu)
+int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
 {
-        struct ring_buffer_per_cpu *cpu_buffer;
+        struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer);
         DEFINE_WAIT(wait);
         struct rb_irq_work *work;
+        int ret = 0;
 
         /*
          * Depending on what the caller is waiting for, either any
@@ -564,36 +566,61 @@ int ring_buffer_wait(struct ring_buffer *buffer, int cpu)
         }
 
 
-        prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE);
+        while (true) {
+                prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE);
 
-        /*
-         * The events can happen in critical sections where
-         * checking a work queue can cause deadlocks.
-         * After adding a task to the queue, this flag is set
-         * only to notify events to try to wake up the queue
-         * using irq_work.
-         *
-         * We don't clear it even if the buffer is no longer
-         * empty. The flag only causes the next event to run
-         * irq_work to do the work queue wake up. The worse
-         * that can happen if we race with !trace_empty() is that
-         * an event will cause an irq_work to try to wake up
-         * an empty queue.
-         *
-         * There's no reason to protect this flag either, as
-         * the work queue and irq_work logic will do the necessary
-         * synchronization for the wake ups. The only thing
-         * that is necessary is that the wake up happens after
-         * a task has been queued. It's OK for spurious wake ups.
-         */
-        work->waiters_pending = true;
+                /*
+                 * The events can happen in critical sections where
+                 * checking a work queue can cause deadlocks.
+                 * After adding a task to the queue, this flag is set
+                 * only to notify events to try to wake up the queue
+                 * using irq_work.
+                 *
+                 * We don't clear it even if the buffer is no longer
+                 * empty. The flag only causes the next event to run
+                 * irq_work to do the work queue wake up. The worse
+                 * that can happen if we race with !trace_empty() is that
+                 * an event will cause an irq_work to try to wake up
+                 * an empty queue.
+                 *
+                 * There's no reason to protect this flag either, as
+                 * the work queue and irq_work logic will do the necessary
+                 * synchronization for the wake ups. The only thing
+                 * that is necessary is that the wake up happens after
+                 * a task has been queued. It's OK for spurious wake ups.
+                 */
+                work->waiters_pending = true;
+
+                if (signal_pending(current)) {
+                        ret = -EINTR;
+                        break;
+                }
+
+                if (cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer))
+                        break;
+
+                if (cpu != RING_BUFFER_ALL_CPUS &&
+                    !ring_buffer_empty_cpu(buffer, cpu)) {
+                        unsigned long flags;
+                        bool pagebusy;
+
+                        if (!full)
+                                break;
+
+                        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
+                        pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
+                        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+
+                        if (!pagebusy)
+                                break;
+                }
 
-        if ((cpu == RING_BUFFER_ALL_CPUS && ring_buffer_empty(buffer)) ||
-            (cpu != RING_BUFFER_ALL_CPUS && ring_buffer_empty_cpu(buffer, cpu)))
                 schedule();
+        }
 
         finish_wait(&work->waiters, &wait);
-        return 0;
+
+        return ret;
 }
 
 /**
@@ -626,8 +653,22 @@ int ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu,
                 work = &cpu_buffer->irq_work;
         }
 
-        work->waiters_pending = true;
         poll_wait(filp, &work->waiters, poll_table);
+        work->waiters_pending = true;
+        /*
+         * There's a tight race between setting the waiters_pending and
+         * checking if the ring buffer is empty.  Once the waiters_pending bit
+         * is set, the next event will wake the task up, but we can get stuck
+         * if there's only a single event in.
+         *
+         * FIXME: Ideally, we need a memory barrier on the writer side as well,
+         * but adding a memory barrier to all events will cause too much of a
+         * performance hit in the fast path.  We only need a memory barrier when
+         * the buffer goes from empty to having content.  But as this race is
+         * extremely small, and it's not a problem if another event comes in, we
+         * will fix it later.
+         */
+        smp_mb();
 
         if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) ||
             (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))
@@ -1968,7 +2009,7 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta)
 
 /**
  * rb_update_event - update event type and data
- * @event: the even to update
+ * @event: the event to update
  * @type: the type of event
  * @length: the size of the event field in the ring buffer
  *
@@ -3341,21 +3382,16 @@ static void rb_iter_reset(struct ring_buffer_iter *iter)
         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
 
         /* Iterator usage is expected to have record disabled */
-        if (list_empty(&cpu_buffer->reader_page->list)) {
-                iter->head_page = rb_set_head_page(cpu_buffer);
-                if (unlikely(!iter->head_page))
-                        return;
-                iter->head = iter->head_page->read;
-        } else {
-                iter->head_page = cpu_buffer->reader_page;
-                iter->head = cpu_buffer->reader_page->read;
-        }
+        iter->head_page = cpu_buffer->reader_page;
+        iter->head = cpu_buffer->reader_page->read;
+
+        iter->cache_reader_page = iter->head_page;
+        iter->cache_read = cpu_buffer->read;
+
         if (iter->head)
                 iter->read_stamp = cpu_buffer->read_stamp;
         else
                 iter->read_stamp = iter->head_page->page->time_stamp;
-        iter->cache_reader_page = cpu_buffer->reader_page;
-        iter->cache_read = cpu_buffer->read;
 }
 
 /**
@@ -3748,12 +3784,14 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
                 return NULL;
 
         /*
-         * We repeat when a time extend is encountered.
-         * Since the time extend is always attached to a data event,
-         * we should never loop more than once.
-         * (We never hit the following condition more than twice).
+         * We repeat when a time extend is encountered or we hit
+         * the end of the page. Since the time extend is always attached
+         * to a data event, we should never loop more than three times.
+         * Once for going to next page, once on time extend, and
+         * finally once to get the event.
+         * (We never hit the following condition more than thrice).
          */
-        if (RB_WARN_ON(cpu_buffer, ++nr_loops > 2))
+        if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3))
                 return NULL;
 
         if (rb_per_cpu_empty(cpu_buffer))