1 files changed, 178 insertions, 56 deletions

diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 41ca394feb22..3632ce87674f 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -319,6 +319,11 @@ EXPORT_SYMBOL_GPL(ring_buffer_event_data);
 #define TS_MASK         ((1ULL << TS_SHIFT) - 1)
 #define TS_DELTA_TEST   (~TS_MASK)
 
+/* Flag when events were overwritten */
+#define RB_MISSED_EVENTS        (1 << 31)
+/* Missed count stored at end */
+#define RB_MISSED_STORED        (1 << 30)
+
 struct buffer_data_page {
         u64              time_stamp;    /* page time stamp */
         local_t          commit;        /* write committed index */
@@ -338,6 +343,7 @@ struct buffer_page {
         local_t          write;         /* index for next write */
         unsigned         read;          /* index for next read */
         local_t          entries;       /* entries on this page */
+        unsigned long    real_end;      /* real end of data */
         struct buffer_data_page *page;  /* Actual data page */
 };
 
@@ -417,6 +423,12 @@ int ring_buffer_print_page_header(struct trace_seq *s)
                                (unsigned int)sizeof(field.commit),
                                (unsigned int)is_signed_type(long));
 
+        ret = trace_seq_printf(s, "\tfield: int overwrite;\t"
+                               "offset:%u;\tsize:%u;\tsigned:%u;\n",
+                               (unsigned int)offsetof(typeof(field), commit),
+                               1,
+                               (unsigned int)is_signed_type(long));
+
         ret = trace_seq_printf(s, "\tfield: char data;\t"
                                "offset:%u;\tsize:%u;\tsigned:%u;\n",
                                (unsigned int)offsetof(typeof(field), data),
@@ -431,6 +443,7 @@ int ring_buffer_print_page_header(struct trace_seq *s)
  */
 struct ring_buffer_per_cpu {
         int                             cpu;
+        atomic_t                        record_disabled;
         struct ring_buffer              *buffer;
         spinlock_t                      reader_lock;    /* serialize readers */
         arch_spinlock_t                 lock;
@@ -440,6 +453,8 @@ struct ring_buffer_per_cpu {
         struct buffer_page              *tail_page;     /* write to tail */
         struct buffer_page              *commit_page;   /* committed pages */
         struct buffer_page              *reader_page;
+        unsigned long                   lost_events;
+        unsigned long                   last_overrun;
         local_t                         commit_overrun;
         local_t                         overrun;
         local_t                         entries;
@@ -448,7 +463,6 @@ struct ring_buffer_per_cpu {
         unsigned long                   read;
         u64                             write_stamp;
         u64                             read_stamp;
-        atomic_t                        record_disabled;
 };
 
 struct ring_buffer {
@@ -1754,6 +1768,14 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
          * must fill the old tail_page with padding.
          */
         if (tail >= BUF_PAGE_SIZE) {
+                /*
+                 * If the page was filled, then we still need
+                 * to update the real_end. Reset it to zero
+                 * and the reader will ignore it.
+                 */
+                if (tail == BUF_PAGE_SIZE)
+                        tail_page->real_end = 0;
+
                 local_sub(length, &tail_page->write);
                 return;
         }
@@ -1762,6 +1784,13 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
         kmemcheck_annotate_bitfield(event, bitfield);
 
         /*
+         * Save the original length to the meta data.
+         * This will be used by the reader to add lost event
+         * counter.
+         */
+        tail_page->real_end = tail;
+
+        /*
          * If this event is bigger than the minimum size, then
          * we need to be careful that we don't subtract the
          * write counter enough to allow another writer to slip
@@ -1979,17 +2008,13 @@ rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
                   u64 *ts, u64 *delta)
 {
         struct ring_buffer_event *event;
-        static int once;
         int ret;
 
-        if (unlikely(*delta > (1ULL << 59) && !once++)) {
-                printk(KERN_WARNING "Delta way too big! %llu"
-                       " ts=%llu write stamp = %llu\n",
-                       (unsigned long long)*delta,
-                       (unsigned long long)*ts,
-                       (unsigned long long)cpu_buffer->write_stamp);
-                WARN_ON(1);
-        }
+        WARN_ONCE(*delta > (1ULL << 59),
+                  KERN_WARNING "Delta way too big! %llu ts=%llu write stamp = %llu\n",
+                  (unsigned long long)*delta,
+                  (unsigned long long)*ts,
+                  (unsigned long long)cpu_buffer->write_stamp);
 
         /*
          * The delta is too big, we to add a
@@ -2217,8 +2242,6 @@ static void trace_recursive_unlock(void)
 
 #endif
 
-static DEFINE_PER_CPU(int, rb_need_resched);
-
 /**
  * ring_buffer_lock_reserve - reserve a part of the buffer
  * @buffer: the ring buffer to reserve from
@@ -2239,13 +2262,13 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
 {
         struct ring_buffer_per_cpu *cpu_buffer;
         struct ring_buffer_event *event;
-        int cpu, resched;
+        int cpu;
 
         if (ring_buffer_flags != RB_BUFFERS_ON)
                 return NULL;
 
         /* If we are tracing schedule, we don't want to recurse */
-        resched = ftrace_preempt_disable();
+        preempt_disable_notrace();
 
         if (atomic_read(&buffer->record_disabled))
                 goto out_nocheck;
@@ -2270,21 +2293,13 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
         if (!event)
                 goto out;
 
-        /*
-         * Need to store resched state on this cpu.
-         * Only the first needs to.
-         */
-
-        if (preempt_count() == 1)
-                per_cpu(rb_need_resched, cpu) = resched;
-
         return event;
 
  out:
         trace_recursive_unlock();
 
  out_nocheck:
-        ftrace_preempt_enable(resched);
+        preempt_enable_notrace();
         return NULL;
 }
 EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
@@ -2330,13 +2345,7 @@ int ring_buffer_unlock_commit(struct ring_buffer *buffer,
 
         trace_recursive_unlock();
 
-        /*
-         * Only the last preempt count needs to restore preemption.
-         */
-        if (preempt_count() == 1)
-                ftrace_preempt_enable(per_cpu(rb_need_resched, cpu));
-        else
-                preempt_enable_no_resched_notrace();
+        preempt_enable_notrace();
 
         return 0;
 }
@@ -2444,13 +2453,7 @@ void ring_buffer_discard_commit(struct ring_buffer *buffer,
 
         trace_recursive_unlock();
 
-        /*
-         * Only the last preempt count needs to restore preemption.
-         */
-        if (preempt_count() == 1)
-                ftrace_preempt_enable(per_cpu(rb_need_resched, cpu));
-        else
-                preempt_enable_no_resched_notrace();
+        preempt_enable_notrace();
 
 }
 EXPORT_SYMBOL_GPL(ring_buffer_discard_commit);
@@ -2476,12 +2479,12 @@ int ring_buffer_write(struct ring_buffer *buffer,
         struct ring_buffer_event *event;
         void *body;
         int ret = -EBUSY;
-        int cpu, resched;
+        int cpu;
 
         if (ring_buffer_flags != RB_BUFFERS_ON)
                 return -EBUSY;
 
-        resched = ftrace_preempt_disable();
+        preempt_disable_notrace();
 
         if (atomic_read(&buffer->record_disabled))
                 goto out;
@@ -2511,7 +2514,7 @@ int ring_buffer_write(struct ring_buffer *buffer,
 
         ret = 0;
  out:
-        ftrace_preempt_enable(resched);
+        preempt_enable_notrace();
 
         return ret;
 }
@@ -2838,6 +2841,7 @@ static struct buffer_page *
 rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
 {
         struct buffer_page *reader = NULL;
+        unsigned long overwrite;
         unsigned long flags;
         int nr_loops = 0;
         int ret;
@@ -2879,6 +2883,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
         local_set(&cpu_buffer->reader_page->write, 0);
         local_set(&cpu_buffer->reader_page->entries, 0);
         local_set(&cpu_buffer->reader_page->page->commit, 0);
+        cpu_buffer->reader_page->real_end = 0;
 
  spin:
         /*
@@ -2899,6 +2904,18 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
         rb_set_list_to_head(cpu_buffer, &cpu_buffer->reader_page->list);
 
         /*
+         * We want to make sure we read the overruns after we set up our
+         * pointers to the next object. The writer side does a
+         * cmpxchg to cross pages which acts as the mb on the writer
+         * side. Note, the reader will constantly fail the swap
+         * while the writer is updating the pointers, so this
+         * guarantees that the overwrite recorded here is the one we
+         * want to compare with the last_overrun.
+         */
+        smp_mb();
+        overwrite = local_read(&(cpu_buffer->overrun));
+
+        /*
          * Here's the tricky part.
          *
          * We need to move the pointer past the header page.
@@ -2929,6 +2946,11 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
         cpu_buffer->reader_page = reader;
         rb_reset_reader_page(cpu_buffer);
 
+        if (overwrite != cpu_buffer->last_overrun) {
+                cpu_buffer->lost_events = overwrite - cpu_buffer->last_overrun;
+                cpu_buffer->last_overrun = overwrite;
+        }
+
         goto again;
 
  out:
@@ -3005,8 +3027,14 @@ static void rb_advance_iter(struct ring_buffer_iter *iter)
                 rb_advance_iter(iter);
 }
 
+static int rb_lost_events(struct ring_buffer_per_cpu *cpu_buffer)
+{
+        return cpu_buffer->lost_events;
+}
+
 static struct ring_buffer_event *
-rb_buffer_peek(struct ring_buffer_per_cpu *cpu_buffer, u64 *ts)
+rb_buffer_peek(struct ring_buffer_per_cpu *cpu_buffer, u64 *ts,
+               unsigned long *lost_events)
 {
         struct ring_buffer_event *event;
         struct buffer_page *reader;
@@ -3058,6 +3086,8 @@ rb_buffer_peek(struct ring_buffer_per_cpu *cpu_buffer, u64 *ts)
                         ring_buffer_normalize_time_stamp(cpu_buffer->buffer,
                                                          cpu_buffer->cpu, ts);
                 }
+                if (lost_events)
+                        *lost_events = rb_lost_events(cpu_buffer);
                 return event;
 
         default:
@@ -3168,12 +3198,14 @@ static inline int rb_ok_to_lock(void)
  * @buffer: The ring buffer to read
  * @cpu: The cpu to peak at
  * @ts: The timestamp counter of this event.
+ * @lost_events: a variable to store if events were lost (may be NULL)
  *
  * This will return the event that will be read next, but does
  * not consume the data.
  */
 struct ring_buffer_event *
-ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
+ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts,
+                 unsigned long *lost_events)
 {
         struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
         struct ring_buffer_event *event;
@@ -3188,7 +3220,7 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
         local_irq_save(flags);
         if (dolock)
                 spin_lock(&cpu_buffer->reader_lock);
-        event = rb_buffer_peek(cpu_buffer, ts);
+        event = rb_buffer_peek(cpu_buffer, ts, lost_events);
         if (event && event->type_len == RINGBUF_TYPE_PADDING)
                 rb_advance_reader(cpu_buffer);
         if (dolock)
@@ -3230,13 +3262,17 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
 /**
  * ring_buffer_consume - return an event and consume it
  * @buffer: The ring buffer to get the next event from
+ * @cpu: the cpu to read the buffer from
+ * @ts: a variable to store the timestamp (may be NULL)
+ * @lost_events: a variable to store if events were lost (may be NULL)
  *
  * Returns the next event in the ring buffer, and that event is consumed.
  * Meaning, that sequential reads will keep returning a different event,
  * and eventually empty the ring buffer if the producer is slower.
  */
 struct ring_buffer_event *
-ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
+ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts,
+                    unsigned long *lost_events)
 {
         struct ring_buffer_per_cpu *cpu_buffer;
         struct ring_buffer_event *event = NULL;
@@ -3257,9 +3293,11 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
         if (dolock)
                 spin_lock(&cpu_buffer->reader_lock);
 
-        event = rb_buffer_peek(cpu_buffer, ts);
-        if (event)
+        event = rb_buffer_peek(cpu_buffer, ts, lost_events);
+        if (event) {
+                cpu_buffer->lost_events = 0;
                 rb_advance_reader(cpu_buffer);
+        }
 
         if (dolock)
                 spin_unlock(&cpu_buffer->reader_lock);
@@ -3276,23 +3314,30 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
 EXPORT_SYMBOL_GPL(ring_buffer_consume);
 
 /**
- * ring_buffer_read_start - start a non consuming read of the buffer
+ * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer
  * @buffer: The ring buffer to read from
  * @cpu: The cpu buffer to iterate over
  *
- * This starts up an iteration through the buffer. It also disables
- * the recording to the buffer until the reading is finished.
- * This prevents the reading from being corrupted. This is not
- * a consuming read, so a producer is not expected.
+ * This performs the initial preparations necessary to iterate
+ * through the buffer.  Memory is allocated, buffer recording
+ * is disabled, and the iterator pointer is returned to the caller.
  *
- * Must be paired with ring_buffer_finish.
+ * Disabling buffer recordng prevents the reading from being
+ * corrupted. This is not a consuming read, so a producer is not
+ * expected.
+ *
+ * After a sequence of ring_buffer_read_prepare calls, the user is
+ * expected to make at least one call to ring_buffer_prepare_sync.
+ * Afterwards, ring_buffer_read_start is invoked to get things going
+ * for real.
+ *
+ * This overall must be paired with ring_buffer_finish.
  */
 struct ring_buffer_iter *
-ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
+ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu)
 {
         struct ring_buffer_per_cpu *cpu_buffer;
         struct ring_buffer_iter *iter;
-        unsigned long flags;
 
         if (!cpumask_test_cpu(cpu, buffer->cpumask))
                 return NULL;
@@ -3306,15 +3351,52 @@ ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
         iter->cpu_buffer = cpu_buffer;
 
         atomic_inc(&cpu_buffer->record_disabled);
+
+        return iter;
+}
+EXPORT_SYMBOL_GPL(ring_buffer_read_prepare);
+
+/**
+ * ring_buffer_read_prepare_sync - Synchronize a set of prepare calls
+ *
+ * All previously invoked ring_buffer_read_prepare calls to prepare
+ * iterators will be synchronized.  Afterwards, read_buffer_read_start
+ * calls on those iterators are allowed.
+ */
+void
+ring_buffer_read_prepare_sync(void)
+{
         synchronize_sched();
+}
+EXPORT_SYMBOL_GPL(ring_buffer_read_prepare_sync);
+
+/**
+ * ring_buffer_read_start - start a non consuming read of the buffer
+ * @iter: The iterator returned by ring_buffer_read_prepare
+ *
+ * This finalizes the startup of an iteration through the buffer.
+ * The iterator comes from a call to ring_buffer_read_prepare and
+ * an intervening ring_buffer_read_prepare_sync must have been
+ * performed.
+ *
+ * Must be paired with ring_buffer_finish.
+ */
+void
+ring_buffer_read_start(struct ring_buffer_iter *iter)
+{
+        struct ring_buffer_per_cpu *cpu_buffer;
+        unsigned long flags;
+
+        if (!iter)
+                return;
+
+        cpu_buffer = iter->cpu_buffer;
 
         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
         arch_spin_lock(&cpu_buffer->lock);
         rb_iter_reset(iter);
         arch_spin_unlock(&cpu_buffer->lock);
         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
-
-        return iter;
 }
 EXPORT_SYMBOL_GPL(ring_buffer_read_start);
 
@@ -3408,6 +3490,9 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
         cpu_buffer->write_stamp = 0;
         cpu_buffer->read_stamp = 0;
 
+        cpu_buffer->lost_events = 0;
+        cpu_buffer->last_overrun = 0;
+
         rb_head_page_activate(cpu_buffer);
 }
 
@@ -3683,6 +3768,7 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
         struct ring_buffer_event *event;
         struct buffer_data_page *bpage;
         struct buffer_page *reader;
+        unsigned long missed_events;
         unsigned long flags;
         unsigned int commit;
         unsigned int read;
@@ -3719,6 +3805,9 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
         read = reader->read;
         commit = rb_page_commit(reader);
 
+        /* Check if any events were dropped */
+        missed_events = cpu_buffer->lost_events;
+
         /*
          * If this page has been partially read or
          * if len is not big enough to read the rest of the page or
@@ -3779,9 +3868,42 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
                 local_set(&reader->entries, 0);
                 reader->read = 0;
                 *data_page = bpage;
+
+                /*
+                 * Use the real_end for the data size,
+                 * This gives us a chance to store the lost events
+                 * on the page.
+                 */
+                if (reader->real_end)
+                        local_set(&bpage->commit, reader->real_end);
         }
         ret = read;
 
+        cpu_buffer->lost_events = 0;
+
+        commit = local_read(&bpage->commit);
+        /*
+         * Set a flag in the commit field if we lost events
+         */
+        if (missed_events) {
+                /* If there is room at the end of the page to save the
+                 * missed events, then record it there.
+                 */
+                if (BUF_PAGE_SIZE - commit >= sizeof(missed_events)) {
+                        memcpy(&bpage->data[commit], &missed_events,
+                               sizeof(missed_events));
+                        local_add(RB_MISSED_STORED, &bpage->commit);
+                        commit += sizeof(missed_events);
+                }
+                local_add(RB_MISSED_EVENTS, &bpage->commit);
+        }
+
+        /*
+         * This page may be off to user land. Zero it out here.
+         */
+        if (commit < BUF_PAGE_SIZE)
+                memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit);
+
  out_unlock:
         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);