Forward to 2.6.33-rc8

Merge branch 'linus' into rt/head with a pile of conflicts.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

1 files changed, 93 insertions, 75 deletions

diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index f780675b54cb..9e095ef33be6 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -201,8 +201,6 @@ int tracing_is_on(void)
 }
 EXPORT_SYMBOL_GPL(tracing_is_on);
 
-#include "trace.h"
-
 #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
 #define RB_ALIGNMENT            4U
 #define RB_MAX_SMALL_DATA       (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
@@ -399,18 +397,21 @@ int ring_buffer_print_page_header(struct trace_seq *s)
         int ret;
 
         ret = trace_seq_printf(s, "\tfield: u64 timestamp;\t"
-                               "offset:0;\tsize:%u;\n",
-                               (unsigned int)sizeof(field.time_stamp));
+                               "offset:0;\tsize:%u;\tsigned:%u;\n",
+                               (unsigned int)sizeof(field.time_stamp),
+                               (unsigned int)is_signed_type(u64));
 
         ret = trace_seq_printf(s, "\tfield: local_t commit;\t"
-                               "offset:%u;\tsize:%u;\n",
+                               "offset:%u;\tsize:%u;\tsigned:%u;\n",
                                (unsigned int)offsetof(typeof(field), commit),
-                               (unsigned int)sizeof(field.commit));
+                               (unsigned int)sizeof(field.commit),
+                               (unsigned int)is_signed_type(long));
 
         ret = trace_seq_printf(s, "\tfield: char data;\t"
-                               "offset:%u;\tsize:%u;\n",
+                               "offset:%u;\tsize:%u;\tsigned:%u;\n",
                                (unsigned int)offsetof(typeof(field), data),
-                               (unsigned int)BUF_PAGE_SIZE);
+                               (unsigned int)BUF_PAGE_SIZE,
+                               (unsigned int)is_signed_type(char));
 
         return ret;
 }
@@ -421,8 +422,8 @@ int ring_buffer_print_page_header(struct trace_seq *s)
 struct ring_buffer_per_cpu {
         int                             cpu;
         struct ring_buffer              *buffer;
-        atomic_spinlock_t               reader_lock;    /* serialize readers */
-        raw_spinlock_t                  lock;
+        raw_spinlock_t                  reader_lock;    /* serialize readers */
+        arch_spinlock_t                 lock;
         struct lock_class_key           lock_key;
         struct list_head                *pages;
         struct buffer_page              *head_page;     /* read from head */
@@ -463,6 +464,8 @@ struct ring_buffer_iter {
         struct ring_buffer_per_cpu      *cpu_buffer;
         unsigned long                   head;
         struct buffer_page              *head_page;
+        struct buffer_page              *cache_reader_page;
+        unsigned long                   cache_read;
         u64                             read_stamp;
 };
 
@@ -485,7 +488,7 @@ struct ring_buffer_iter {
 /* Up this if you want to test the TIME_EXTENTS and normalization */
 #define DEBUG_SHIFT 0
 
-static inline u64 rb_time_stamp(struct ring_buffer *buffer, int cpu)
+static inline u64 rb_time_stamp(struct ring_buffer *buffer)
 {
         /* shift to debug/test normalization and TIME_EXTENTS */
         return buffer->clock() << DEBUG_SHIFT;
@@ -496,7 +499,7 @@ u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu)
         u64 time;
 
         preempt_disable_notrace();
-        time = rb_time_stamp(buffer, cpu);
+        time = rb_time_stamp(buffer);
         preempt_enable_no_resched_notrace();
 
         return time;
@@ -601,7 +604,7 @@ static struct list_head *rb_list_head(struct list_head *list)
 }
 
 /*
- * rb_is_head_page - test if the give page is the head page
+ * rb_is_head_page - test if the given page is the head page
  *
  * Because the reader may move the head_page pointer, we can
  * not trust what the head page is (it may be pointing to
@@ -995,9 +998,9 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
 
         cpu_buffer->cpu = cpu;
         cpu_buffer->buffer = buffer;
-        atomic_spin_lock_init(&cpu_buffer->reader_lock);
+        raw_spin_lock_init(&cpu_buffer->reader_lock);
         lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);
-        cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
+        cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
 
         bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
                             GFP_KERNEL, cpu_to_node(cpu));
@@ -1188,13 +1191,12 @@ static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
 static void
 rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
 {
-        struct buffer_page *bpage;
+        struct buffer_page *bpage, *tmp;
         struct list_head *p;
+        LIST_HEAD(tofree);
         unsigned i;
 
-        atomic_inc(&cpu_buffer->record_disabled);
-        synchronize_sched();
-
+        raw_spin_lock_irq(&cpu_buffer->reader_lock);
         rb_head_page_deactivate(cpu_buffer);
 
         for (i = 0; i < nr_pages; i++) {
@@ -1202,17 +1204,21 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
                         return;
                 p = cpu_buffer->pages->next;
                 bpage = list_entry(p, struct buffer_page, list);
-                list_del_init(&bpage->list);
-                free_buffer_page(bpage);
+                list_del(&bpage->list);
+                list_add(&bpage->list, &tofree);
         }
         if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)))
                 return;
 
         rb_reset_cpu(cpu_buffer);
-
         rb_check_pages(cpu_buffer);
 
-        atomic_dec(&cpu_buffer->record_disabled);
+        raw_spin_unlock_irq(&cpu_buffer->reader_lock);
+
+        list_for_each_entry_safe(bpage, tmp, &tofree, list) {
+                list_del_init(&bpage->list);
+                free_buffer_page(bpage);
+        }
 
 }
 
@@ -1224,10 +1230,7 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
         struct list_head *p;
         unsigned i;
 
-        atomic_inc(&cpu_buffer->record_disabled);
-        synchronize_sched();
-
-        atomic_spin_lock_irq(&cpu_buffer->reader_lock);
+        raw_spin_lock_irq(&cpu_buffer->reader_lock);
         rb_head_page_deactivate(cpu_buffer);
 
         for (i = 0; i < nr_pages; i++) {
@@ -1239,11 +1242,9 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
                 list_add_tail(&bpage->list, cpu_buffer->pages);
         }
         rb_reset_cpu(cpu_buffer);
-        atomic_spin_unlock_irq(&cpu_buffer->reader_lock);
-
         rb_check_pages(cpu_buffer);
 
-        atomic_dec(&cpu_buffer->record_disabled);
+        raw_spin_unlock_irq(&cpu_buffer->reader_lock);
 }
 
 /**
@@ -1251,11 +1252,6 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
  * @buffer: the buffer to resize.
  * @size: the new size.
  *
- * The tracer is responsible for making sure that the buffer is
- * not being used while changing the size.
- * Note: We may be able to change the above requirement by using
- *  RCU synchronizations.
- *
  * Minimum size is 2 * BUF_PAGE_SIZE.
  *
  * Returns -1 on failure.
@@ -1287,6 +1283,11 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
         if (size == buffer_size)
                 return size;
 
+        atomic_inc(&buffer->record_disabled);
+
+        /* Make sure all writers are done with this buffer. */
+        synchronize_sched();
+
         mutex_lock(&buffer->mutex);
         get_online_cpus();
 
@@ -1349,6 +1350,8 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
         put_online_cpus();
         mutex_unlock(&buffer->mutex);
 
+        atomic_dec(&buffer->record_disabled);
+
         return size;
 
  free_pages:
@@ -1358,6 +1361,7 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
         }
         put_online_cpus();
         mutex_unlock(&buffer->mutex);
+        atomic_dec(&buffer->record_disabled);
         return -ENOMEM;
 
         /*
@@ -1367,6 +1371,7 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
  out_fail:
         put_online_cpus();
         mutex_unlock(&buffer->mutex);
+        atomic_dec(&buffer->record_disabled);
         return -1;
 }
 EXPORT_SYMBOL_GPL(ring_buffer_resize);
@@ -1787,9 +1792,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
 static struct ring_buffer_event *
 rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
              unsigned long length, unsigned long tail,
-             struct buffer_page *commit_page,
              struct buffer_page *tail_page, u64 *ts)
 {
+        struct buffer_page *commit_page = cpu_buffer->commit_page;
         struct ring_buffer *buffer = cpu_buffer->buffer;
         struct buffer_page *next_page;
         int ret;
@@ -1870,7 +1875,7 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
                  * Nested commits always have zero deltas, so
                  * just reread the time stamp
                  */
-                *ts = rb_time_stamp(buffer, cpu_buffer->cpu);
+                *ts = rb_time_stamp(buffer);
                 next_page->page->time_stamp = *ts;
         }
 
@@ -1892,13 +1897,10 @@ static struct ring_buffer_event *
 __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
                   unsigned type, unsigned long length, u64 *ts)
 {
-        struct buffer_page *tail_page, *commit_page;
+        struct buffer_page *tail_page;
         struct ring_buffer_event *event;
         unsigned long tail, write;
 
-        commit_page = cpu_buffer->commit_page;
-        /* we just need to protect against interrupts */
-        barrier();
         tail_page = cpu_buffer->tail_page;
         write = local_add_return(length, &tail_page->write);
 
@@ -1909,7 +1911,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
         /* See if we shot pass the end of this buffer page */
         if (write > BUF_PAGE_SIZE)
                 return rb_move_tail(cpu_buffer, length, tail,
-                                    commit_page, tail_page, ts);
+                                    tail_page, ts);
 
         /* We reserved something on the buffer */
 
@@ -2113,7 +2115,7 @@ rb_reserve_next_event(struct ring_buffer *buffer,
         if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
                 goto out_fail;
 
-        ts = rb_time_stamp(cpu_buffer->buffer, cpu_buffer->cpu);
+        ts = rb_time_stamp(cpu_buffer->buffer);
 
         /*
          * Only the first commit can update the timestamp.
@@ -2683,7 +2685,7 @@ unsigned long ring_buffer_entries(struct ring_buffer *buffer)
 EXPORT_SYMBOL_GPL(ring_buffer_entries);
 
 /**
- * ring_buffer_overrun_cpu - get the number of overruns in buffer
+ * ring_buffer_overruns - get the number of overruns in buffer
  * @buffer: The ring buffer
  *
  * Returns the total number of overruns in the ring buffer
@@ -2723,6 +2725,8 @@ static void rb_iter_reset(struct ring_buffer_iter *iter)
                 iter->read_stamp = cpu_buffer->read_stamp;
         else
                 iter->read_stamp = iter->head_page->page->time_stamp;
+        iter->cache_reader_page = cpu_buffer->reader_page;
+        iter->cache_read = cpu_buffer->read;
 }
 
 /**
@@ -2742,9 +2746,9 @@ void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
 
         cpu_buffer = iter->cpu_buffer;
 
-        atomic_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
+        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
         rb_iter_reset(iter);
-        atomic_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
 }
 EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
 
@@ -2834,7 +2838,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
         int ret;
 
         local_irq_save(flags);
-        __raw_spin_lock(&cpu_buffer->lock);
+        arch_spin_lock(&cpu_buffer->lock);
 
  again:
         /*
@@ -2876,7 +2880,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
          * Splice the empty reader page into the list around the head.
          */
         reader = rb_set_head_page(cpu_buffer);
-        cpu_buffer->reader_page->list.next = reader->list.next;
+        cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);
         cpu_buffer->reader_page->list.prev = reader->list.prev;
 
         /*
@@ -2913,7 +2917,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
          *
          * Now make the new head point back to the reader page.
          */
-        reader->list.next->prev = &cpu_buffer->reader_page->list;
+        rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
         rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
 
         /* Finally update the reader page to the new head */
@@ -2923,7 +2927,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
         goto again;
 
  out:
-        __raw_spin_unlock(&cpu_buffer->lock);
+        arch_spin_unlock(&cpu_buffer->lock);
         local_irq_restore(flags);
 
         return reader;
@@ -3067,13 +3071,22 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
         struct ring_buffer_event *event;
         int nr_loops = 0;
 
-        if (ring_buffer_iter_empty(iter))
-                return NULL;
-
         cpu_buffer = iter->cpu_buffer;
         buffer = cpu_buffer->buffer;
 
+        /*
+         * Check if someone performed a consuming read to
+         * the buffer. A consuming read invalidates the iterator
+         * and we need to reset the iterator in this case.
+         */
+        if (unlikely(iter->cache_read != cpu_buffer->read ||
+                     iter->cache_reader_page != cpu_buffer->reader_page))
+                rb_iter_reset(iter);
+
  again:
+        if (ring_buffer_iter_empty(iter))
+                return NULL;
+
         /*
          * We repeat when a timestamp is encountered.
          * We can get multiple timestamps by nested interrupts or also
@@ -3088,6 +3101,11 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
         if (rb_per_cpu_empty(cpu_buffer))
                 return NULL;
 
+        if (iter->head >= local_read(&iter->head_page->page->commit)) {
+                rb_inc_iter(iter);
+                goto again;
+        }
+
         event = rb_iter_head_event(iter);
 
         switch (event->type_len) {
@@ -3164,12 +3182,12 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
  again:
         local_irq_save(flags);
         if (dolock)
-                atomic_spin_lock(&cpu_buffer->reader_lock);
+                raw_spin_lock(&cpu_buffer->reader_lock);
         event = rb_buffer_peek(cpu_buffer, ts);
         if (event && event->type_len == RINGBUF_TYPE_PADDING)
                 rb_advance_reader(cpu_buffer);
         if (dolock)
-                atomic_spin_unlock(&cpu_buffer->reader_lock);
+                raw_spin_unlock(&cpu_buffer->reader_lock);
         local_irq_restore(flags);
 
         if (event && event->type_len == RINGBUF_TYPE_PADDING)
@@ -3194,9 +3212,9 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
         unsigned long flags;
 
  again:
-        atomic_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
+        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
         event = rb_iter_peek(iter, ts);
-        atomic_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
 
         if (event && event->type_len == RINGBUF_TYPE_PADDING)
                 goto again;
@@ -3232,14 +3250,14 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
         cpu_buffer = buffer->buffers[cpu];
         local_irq_save(flags);
         if (dolock)
-                atomic_spin_lock(&cpu_buffer->reader_lock);
+                raw_spin_lock(&cpu_buffer->reader_lock);
 
         event = rb_buffer_peek(cpu_buffer, ts);
         if (event)
                 rb_advance_reader(cpu_buffer);
 
         if (dolock)
-                atomic_spin_unlock(&cpu_buffer->reader_lock);
+                raw_spin_unlock(&cpu_buffer->reader_lock);
         local_irq_restore(flags);
 
  out:
@@ -3285,11 +3303,11 @@ ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
         atomic_inc(&cpu_buffer->record_disabled);
         synchronize_sched();
 
-        atomic_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
-        __raw_spin_lock(&cpu_buffer->lock);
+        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
+        arch_spin_lock(&cpu_buffer->lock);
         rb_iter_reset(iter);
-        __raw_spin_unlock(&cpu_buffer->lock);
-        atomic_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+        arch_spin_unlock(&cpu_buffer->lock);
+        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
 
         return iter;
 }
@@ -3326,7 +3344,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
         unsigned long flags;
 
-        atomic_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
+        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  again:
         event = rb_iter_peek(iter, ts);
         if (!event)
@@ -3337,7 +3355,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
 
         rb_advance_iter(iter);
  out:
-        atomic_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
 
         return event;
 }
@@ -3403,19 +3421,19 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
 
         atomic_inc(&cpu_buffer->record_disabled);
 
-        atomic_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
+        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
 
         if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))
                 goto out;
 
-        __raw_spin_lock(&cpu_buffer->lock);
+        arch_spin_lock(&cpu_buffer->lock);
 
         rb_reset_cpu(cpu_buffer);
 
-        __raw_spin_unlock(&cpu_buffer->lock);
+        arch_spin_unlock(&cpu_buffer->lock);
 
  out:
-        atomic_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
 
         atomic_dec(&cpu_buffer->record_disabled);
 }
@@ -3453,10 +3471,10 @@ int ring_buffer_empty(struct ring_buffer *buffer)
                 cpu_buffer = buffer->buffers[cpu];
                 local_irq_save(flags);
                 if (dolock)
-                        atomic_spin_lock(&cpu_buffer->reader_lock);
+                        raw_spin_lock(&cpu_buffer->reader_lock);
                 ret = rb_per_cpu_empty(cpu_buffer);
                 if (dolock)
-                        atomic_spin_unlock(&cpu_buffer->reader_lock);
+                        raw_spin_unlock(&cpu_buffer->reader_lock);
                 local_irq_restore(flags);
 
                 if (!ret)
@@ -3487,10 +3505,10 @@ int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
         cpu_buffer = buffer->buffers[cpu];
         local_irq_save(flags);
         if (dolock)
-                atomic_spin_lock(&cpu_buffer->reader_lock);
+                raw_spin_lock(&cpu_buffer->reader_lock);
         ret = rb_per_cpu_empty(cpu_buffer);
         if (dolock)
-                atomic_spin_unlock(&cpu_buffer->reader_lock);
+                raw_spin_unlock(&cpu_buffer->reader_lock);
         local_irq_restore(flags);
 
         return ret;
@@ -3685,7 +3703,7 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
         if (!bpage)
                 goto out;
 
-        atomic_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
+        raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
 
         reader = rb_get_reader_page(cpu_buffer);
         if (!reader)
@@ -3760,7 +3778,7 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
         ret = read;
 
  out_unlock:
-        atomic_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
+        raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
 
  out:
         return ret;