diff options
Diffstat (limited to 'kernel/trace/ring_buffer.c')
-rw-r--r-- | kernel/trace/ring_buffer.c | 23 |
1 files changed, 17 insertions, 6 deletions
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 05a9f83b8819..41ca394feb22 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
@@ -14,6 +14,7 @@ | |||
14 | #include <linux/module.h> | 14 | #include <linux/module.h> |
15 | #include <linux/percpu.h> | 15 | #include <linux/percpu.h> |
16 | #include <linux/mutex.h> | 16 | #include <linux/mutex.h> |
17 | #include <linux/slab.h> | ||
17 | #include <linux/init.h> | 18 | #include <linux/init.h> |
18 | #include <linux/hash.h> | 19 | #include <linux/hash.h> |
19 | #include <linux/list.h> | 20 | #include <linux/list.h> |
@@ -207,6 +208,14 @@ EXPORT_SYMBOL_GPL(tracing_is_on); | |||
207 | #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) | 208 | #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) |
208 | #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ | 209 | #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ |
209 | 210 | ||
211 | #if !defined(CONFIG_64BIT) || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) | ||
212 | # define RB_FORCE_8BYTE_ALIGNMENT 0 | ||
213 | # define RB_ARCH_ALIGNMENT RB_ALIGNMENT | ||
214 | #else | ||
215 | # define RB_FORCE_8BYTE_ALIGNMENT 1 | ||
216 | # define RB_ARCH_ALIGNMENT 8U | ||
217 | #endif | ||
218 | |||
210 | /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */ | 219 | /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */ |
211 | #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX | 220 | #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX |
212 | 221 | ||
@@ -1201,18 +1210,19 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) | |||
1201 | 1210 | ||
1202 | for (i = 0; i < nr_pages; i++) { | 1211 | for (i = 0; i < nr_pages; i++) { |
1203 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) | 1212 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) |
1204 | return; | 1213 | goto out; |
1205 | p = cpu_buffer->pages->next; | 1214 | p = cpu_buffer->pages->next; |
1206 | bpage = list_entry(p, struct buffer_page, list); | 1215 | bpage = list_entry(p, struct buffer_page, list); |
1207 | list_del_init(&bpage->list); | 1216 | list_del_init(&bpage->list); |
1208 | free_buffer_page(bpage); | 1217 | free_buffer_page(bpage); |
1209 | } | 1218 | } |
1210 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) | 1219 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) |
1211 | return; | 1220 | goto out; |
1212 | 1221 | ||
1213 | rb_reset_cpu(cpu_buffer); | 1222 | rb_reset_cpu(cpu_buffer); |
1214 | rb_check_pages(cpu_buffer); | 1223 | rb_check_pages(cpu_buffer); |
1215 | 1224 | ||
1225 | out: | ||
1216 | spin_unlock_irq(&cpu_buffer->reader_lock); | 1226 | spin_unlock_irq(&cpu_buffer->reader_lock); |
1217 | } | 1227 | } |
1218 | 1228 | ||
@@ -1229,7 +1239,7 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer, | |||
1229 | 1239 | ||
1230 | for (i = 0; i < nr_pages; i++) { | 1240 | for (i = 0; i < nr_pages; i++) { |
1231 | if (RB_WARN_ON(cpu_buffer, list_empty(pages))) | 1241 | if (RB_WARN_ON(cpu_buffer, list_empty(pages))) |
1232 | return; | 1242 | goto out; |
1233 | p = pages->next; | 1243 | p = pages->next; |
1234 | bpage = list_entry(p, struct buffer_page, list); | 1244 | bpage = list_entry(p, struct buffer_page, list); |
1235 | list_del_init(&bpage->list); | 1245 | list_del_init(&bpage->list); |
@@ -1238,6 +1248,7 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer, | |||
1238 | rb_reset_cpu(cpu_buffer); | 1248 | rb_reset_cpu(cpu_buffer); |
1239 | rb_check_pages(cpu_buffer); | 1249 | rb_check_pages(cpu_buffer); |
1240 | 1250 | ||
1251 | out: | ||
1241 | spin_unlock_irq(&cpu_buffer->reader_lock); | 1252 | spin_unlock_irq(&cpu_buffer->reader_lock); |
1242 | } | 1253 | } |
1243 | 1254 | ||
@@ -1547,7 +1558,7 @@ rb_update_event(struct ring_buffer_event *event, | |||
1547 | 1558 | ||
1548 | case 0: | 1559 | case 0: |
1549 | length -= RB_EVNT_HDR_SIZE; | 1560 | length -= RB_EVNT_HDR_SIZE; |
1550 | if (length > RB_MAX_SMALL_DATA) | 1561 | if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) |
1551 | event->array[0] = length; | 1562 | event->array[0] = length; |
1552 | else | 1563 | else |
1553 | event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT); | 1564 | event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT); |
@@ -1722,11 +1733,11 @@ static unsigned rb_calculate_event_length(unsigned length) | |||
1722 | if (!length) | 1733 | if (!length) |
1723 | length = 1; | 1734 | length = 1; |
1724 | 1735 | ||
1725 | if (length > RB_MAX_SMALL_DATA) | 1736 | if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) |
1726 | length += sizeof(event.array[0]); | 1737 | length += sizeof(event.array[0]); |
1727 | 1738 | ||
1728 | length += RB_EVNT_HDR_SIZE; | 1739 | length += RB_EVNT_HDR_SIZE; |
1729 | length = ALIGN(length, RB_ALIGNMENT); | 1740 | length = ALIGN(length, RB_ARCH_ALIGNMENT); |
1730 | 1741 | ||
1731 | return length; | 1742 | return length; |
1732 | } | 1743 | } |