diff options
author | Jonathan Herman <hermanjl@cs.unc.edu> | 2013-01-17 16:15:55 -0500 |
---|---|---|
committer | Jonathan Herman <hermanjl@cs.unc.edu> | 2013-01-17 16:15:55 -0500 |
commit | 8dea78da5cee153b8af9c07a2745f6c55057fe12 (patch) | |
tree | a8f4d49d63b1ecc92f2fddceba0655b2472c5bd9 /kernel/trace/ring_buffer.c | |
parent | 406089d01562f1e2bf9f089fd7637009ebaad589 (diff) |
Patched in Tegra support.
Diffstat (limited to 'kernel/trace/ring_buffer.c')
-rw-r--r-- | kernel/trace/ring_buffer.c | 934 |
1 files changed, 297 insertions, 637 deletions
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index ce8514feedc..731201bf4ac 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
@@ -23,8 +23,6 @@ | |||
23 | #include <asm/local.h> | 23 | #include <asm/local.h> |
24 | #include "trace.h" | 24 | #include "trace.h" |
25 | 25 | ||
26 | static void update_pages_handler(struct work_struct *work); | ||
27 | |||
28 | /* | 26 | /* |
29 | * The ring buffer header is special. We must manually up keep it. | 27 | * The ring buffer header is special. We must manually up keep it. |
30 | */ | 28 | */ |
@@ -156,12 +154,35 @@ enum { | |||
156 | 154 | ||
157 | static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; | 155 | static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; |
158 | 156 | ||
159 | /* Used for individual buffers (after the counter) */ | ||
160 | #define RB_BUFFER_OFF (1 << 20) | ||
161 | |||
162 | #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data) | 157 | #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data) |
163 | 158 | ||
164 | /** | 159 | /** |
160 | * tracing_on - enable all tracing buffers | ||
161 | * | ||
162 | * This function enables all tracing buffers that may have been | ||
163 | * disabled with tracing_off. | ||
164 | */ | ||
165 | void tracing_on(void) | ||
166 | { | ||
167 | set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags); | ||
168 | } | ||
169 | EXPORT_SYMBOL_GPL(tracing_on); | ||
170 | |||
171 | /** | ||
172 | * tracing_off - turn off all tracing buffers | ||
173 | * | ||
174 | * This function stops all tracing buffers from recording data. | ||
175 | * It does not disable any overhead the tracers themselves may | ||
176 | * be causing. This function simply causes all recording to | ||
177 | * the ring buffers to fail. | ||
178 | */ | ||
179 | void tracing_off(void) | ||
180 | { | ||
181 | clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags); | ||
182 | } | ||
183 | EXPORT_SYMBOL_GPL(tracing_off); | ||
184 | |||
185 | /** | ||
165 | * tracing_off_permanent - permanently disable ring buffers | 186 | * tracing_off_permanent - permanently disable ring buffers |
166 | * | 187 | * |
167 | * This function, once called, will disable all ring buffers | 188 | * This function, once called, will disable all ring buffers |
@@ -172,6 +193,15 @@ void tracing_off_permanent(void) | |||
172 | set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags); | 193 | set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags); |
173 | } | 194 | } |
174 | 195 | ||
196 | /** | ||
197 | * tracing_is_on - show state of ring buffers enabled | ||
198 | */ | ||
199 | int tracing_is_on(void) | ||
200 | { | ||
201 | return ring_buffer_flags == RB_BUFFERS_ON; | ||
202 | } | ||
203 | EXPORT_SYMBOL_GPL(tracing_is_on); | ||
204 | |||
175 | #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array)) | 205 | #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array)) |
176 | #define RB_ALIGNMENT 4U | 206 | #define RB_ALIGNMENT 4U |
177 | #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) | 207 | #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) |
@@ -448,10 +478,9 @@ struct ring_buffer_per_cpu { | |||
448 | int cpu; | 478 | int cpu; |
449 | atomic_t record_disabled; | 479 | atomic_t record_disabled; |
450 | struct ring_buffer *buffer; | 480 | struct ring_buffer *buffer; |
451 | raw_spinlock_t reader_lock; /* serialize readers */ | 481 | spinlock_t reader_lock; /* serialize readers */ |
452 | arch_spinlock_t lock; | 482 | arch_spinlock_t lock; |
453 | struct lock_class_key lock_key; | 483 | struct lock_class_key lock_key; |
454 | unsigned int nr_pages; | ||
455 | struct list_head *pages; | 484 | struct list_head *pages; |
456 | struct buffer_page *head_page; /* read from head */ | 485 | struct buffer_page *head_page; /* read from head */ |
457 | struct buffer_page *tail_page; /* write to tail */ | 486 | struct buffer_page *tail_page; /* write to tail */ |
@@ -459,29 +488,21 @@ struct ring_buffer_per_cpu { | |||
459 | struct buffer_page *reader_page; | 488 | struct buffer_page *reader_page; |
460 | unsigned long lost_events; | 489 | unsigned long lost_events; |
461 | unsigned long last_overrun; | 490 | unsigned long last_overrun; |
462 | local_t entries_bytes; | ||
463 | local_t entries; | ||
464 | local_t overrun; | ||
465 | local_t commit_overrun; | 491 | local_t commit_overrun; |
466 | local_t dropped_events; | 492 | local_t overrun; |
493 | local_t entries; | ||
467 | local_t committing; | 494 | local_t committing; |
468 | local_t commits; | 495 | local_t commits; |
469 | unsigned long read; | 496 | unsigned long read; |
470 | unsigned long read_bytes; | ||
471 | u64 write_stamp; | 497 | u64 write_stamp; |
472 | u64 read_stamp; | 498 | u64 read_stamp; |
473 | /* ring buffer pages to update, > 0 to add, < 0 to remove */ | ||
474 | int nr_pages_to_update; | ||
475 | struct list_head new_pages; /* new pages to add */ | ||
476 | struct work_struct update_pages_work; | ||
477 | struct completion update_done; | ||
478 | }; | 499 | }; |
479 | 500 | ||
480 | struct ring_buffer { | 501 | struct ring_buffer { |
502 | unsigned pages; | ||
481 | unsigned flags; | 503 | unsigned flags; |
482 | int cpus; | 504 | int cpus; |
483 | atomic_t record_disabled; | 505 | atomic_t record_disabled; |
484 | atomic_t resize_disabled; | ||
485 | cpumask_var_t cpumask; | 506 | cpumask_var_t cpumask; |
486 | 507 | ||
487 | struct lock_class_key *reader_lock_key; | 508 | struct lock_class_key *reader_lock_key; |
@@ -946,10 +967,6 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) | |||
946 | struct list_head *head = cpu_buffer->pages; | 967 | struct list_head *head = cpu_buffer->pages; |
947 | struct buffer_page *bpage, *tmp; | 968 | struct buffer_page *bpage, *tmp; |
948 | 969 | ||
949 | /* Reset the head page if it exists */ | ||
950 | if (cpu_buffer->head_page) | ||
951 | rb_set_head_page(cpu_buffer); | ||
952 | |||
953 | rb_head_page_deactivate(cpu_buffer); | 970 | rb_head_page_deactivate(cpu_buffer); |
954 | 971 | ||
955 | if (RB_WARN_ON(cpu_buffer, head->next->prev != head)) | 972 | if (RB_WARN_ON(cpu_buffer, head->next->prev != head)) |
@@ -976,10 +993,14 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) | |||
976 | return 0; | 993 | return 0; |
977 | } | 994 | } |
978 | 995 | ||
979 | static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu) | 996 | static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, |
997 | unsigned nr_pages) | ||
980 | { | 998 | { |
981 | int i; | ||
982 | struct buffer_page *bpage, *tmp; | 999 | struct buffer_page *bpage, *tmp; |
1000 | LIST_HEAD(pages); | ||
1001 | unsigned i; | ||
1002 | |||
1003 | WARN_ON(!nr_pages); | ||
983 | 1004 | ||
984 | for (i = 0; i < nr_pages; i++) { | 1005 | for (i = 0; i < nr_pages; i++) { |
985 | struct page *page; | 1006 | struct page *page; |
@@ -990,13 +1011,15 @@ static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu) | |||
990 | */ | 1011 | */ |
991 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), | 1012 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), |
992 | GFP_KERNEL | __GFP_NORETRY, | 1013 | GFP_KERNEL | __GFP_NORETRY, |
993 | cpu_to_node(cpu)); | 1014 | cpu_to_node(cpu_buffer->cpu)); |
994 | if (!bpage) | 1015 | if (!bpage) |
995 | goto free_pages; | 1016 | goto free_pages; |
996 | 1017 | ||
997 | list_add(&bpage->list, pages); | 1018 | rb_check_bpage(cpu_buffer, bpage); |
998 | 1019 | ||
999 | page = alloc_pages_node(cpu_to_node(cpu), | 1020 | list_add(&bpage->list, &pages); |
1021 | |||
1022 | page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), | ||
1000 | GFP_KERNEL | __GFP_NORETRY, 0); | 1023 | GFP_KERNEL | __GFP_NORETRY, 0); |
1001 | if (!page) | 1024 | if (!page) |
1002 | goto free_pages; | 1025 | goto free_pages; |
@@ -1004,27 +1027,6 @@ static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu) | |||
1004 | rb_init_page(bpage->page); | 1027 | rb_init_page(bpage->page); |
1005 | } | 1028 | } |
1006 | 1029 | ||
1007 | return 0; | ||
1008 | |||
1009 | free_pages: | ||
1010 | list_for_each_entry_safe(bpage, tmp, pages, list) { | ||
1011 | list_del_init(&bpage->list); | ||
1012 | free_buffer_page(bpage); | ||
1013 | } | ||
1014 | |||
1015 | return -ENOMEM; | ||
1016 | } | ||
1017 | |||
1018 | static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, | ||
1019 | unsigned nr_pages) | ||
1020 | { | ||
1021 | LIST_HEAD(pages); | ||
1022 | |||
1023 | WARN_ON(!nr_pages); | ||
1024 | |||
1025 | if (__rb_allocate_pages(nr_pages, &pages, cpu_buffer->cpu)) | ||
1026 | return -ENOMEM; | ||
1027 | |||
1028 | /* | 1030 | /* |
1029 | * The ring buffer page list is a circular list that does not | 1031 | * The ring buffer page list is a circular list that does not |
1030 | * start and end with a list head. All page list items point to | 1032 | * start and end with a list head. All page list items point to |
@@ -1033,15 +1035,20 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, | |||
1033 | cpu_buffer->pages = pages.next; | 1035 | cpu_buffer->pages = pages.next; |
1034 | list_del(&pages); | 1036 | list_del(&pages); |
1035 | 1037 | ||
1036 | cpu_buffer->nr_pages = nr_pages; | ||
1037 | |||
1038 | rb_check_pages(cpu_buffer); | 1038 | rb_check_pages(cpu_buffer); |
1039 | 1039 | ||
1040 | return 0; | 1040 | return 0; |
1041 | |||
1042 | free_pages: | ||
1043 | list_for_each_entry_safe(bpage, tmp, &pages, list) { | ||
1044 | list_del_init(&bpage->list); | ||
1045 | free_buffer_page(bpage); | ||
1046 | } | ||
1047 | return -ENOMEM; | ||
1041 | } | 1048 | } |
1042 | 1049 | ||
1043 | static struct ring_buffer_per_cpu * | 1050 | static struct ring_buffer_per_cpu * |
1044 | rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu) | 1051 | rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) |
1045 | { | 1052 | { |
1046 | struct ring_buffer_per_cpu *cpu_buffer; | 1053 | struct ring_buffer_per_cpu *cpu_buffer; |
1047 | struct buffer_page *bpage; | 1054 | struct buffer_page *bpage; |
@@ -1055,11 +1062,9 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu) | |||
1055 | 1062 | ||
1056 | cpu_buffer->cpu = cpu; | 1063 | cpu_buffer->cpu = cpu; |
1057 | cpu_buffer->buffer = buffer; | 1064 | cpu_buffer->buffer = buffer; |
1058 | raw_spin_lock_init(&cpu_buffer->reader_lock); | 1065 | spin_lock_init(&cpu_buffer->reader_lock); |
1059 | lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); | 1066 | lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); |
1060 | cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; | 1067 | cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
1061 | INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler); | ||
1062 | init_completion(&cpu_buffer->update_done); | ||
1063 | 1068 | ||
1064 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), | 1069 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), |
1065 | GFP_KERNEL, cpu_to_node(cpu)); | 1070 | GFP_KERNEL, cpu_to_node(cpu)); |
@@ -1076,9 +1081,8 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu) | |||
1076 | rb_init_page(bpage->page); | 1081 | rb_init_page(bpage->page); |
1077 | 1082 | ||
1078 | INIT_LIST_HEAD(&cpu_buffer->reader_page->list); | 1083 | INIT_LIST_HEAD(&cpu_buffer->reader_page->list); |
1079 | INIT_LIST_HEAD(&cpu_buffer->new_pages); | ||
1080 | 1084 | ||
1081 | ret = rb_allocate_pages(cpu_buffer, nr_pages); | 1085 | ret = rb_allocate_pages(cpu_buffer, buffer->pages); |
1082 | if (ret < 0) | 1086 | if (ret < 0) |
1083 | goto fail_free_reader; | 1087 | goto fail_free_reader; |
1084 | 1088 | ||
@@ -1139,7 +1143,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, | |||
1139 | { | 1143 | { |
1140 | struct ring_buffer *buffer; | 1144 | struct ring_buffer *buffer; |
1141 | int bsize; | 1145 | int bsize; |
1142 | int cpu, nr_pages; | 1146 | int cpu; |
1143 | 1147 | ||
1144 | /* keep it in its own cache line */ | 1148 | /* keep it in its own cache line */ |
1145 | buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), | 1149 | buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), |
@@ -1150,14 +1154,14 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, | |||
1150 | if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) | 1154 | if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) |
1151 | goto fail_free_buffer; | 1155 | goto fail_free_buffer; |
1152 | 1156 | ||
1153 | nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); | 1157 | buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); |
1154 | buffer->flags = flags; | 1158 | buffer->flags = flags; |
1155 | buffer->clock = trace_clock_local; | 1159 | buffer->clock = trace_clock_local; |
1156 | buffer->reader_lock_key = key; | 1160 | buffer->reader_lock_key = key; |
1157 | 1161 | ||
1158 | /* need at least two pages */ | 1162 | /* need at least two pages */ |
1159 | if (nr_pages < 2) | 1163 | if (buffer->pages < 2) |
1160 | nr_pages = 2; | 1164 | buffer->pages = 2; |
1161 | 1165 | ||
1162 | /* | 1166 | /* |
1163 | * In case of non-hotplug cpu, if the ring-buffer is allocated | 1167 | * In case of non-hotplug cpu, if the ring-buffer is allocated |
@@ -1180,7 +1184,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, | |||
1180 | 1184 | ||
1181 | for_each_buffer_cpu(buffer, cpu) { | 1185 | for_each_buffer_cpu(buffer, cpu) { |
1182 | buffer->buffers[cpu] = | 1186 | buffer->buffers[cpu] = |
1183 | rb_allocate_cpu_buffer(buffer, nr_pages, cpu); | 1187 | rb_allocate_cpu_buffer(buffer, cpu); |
1184 | if (!buffer->buffers[cpu]) | 1188 | if (!buffer->buffers[cpu]) |
1185 | goto fail_free_buffers; | 1189 | goto fail_free_buffers; |
1186 | } | 1190 | } |
@@ -1248,223 +1252,58 @@ void ring_buffer_set_clock(struct ring_buffer *buffer, | |||
1248 | 1252 | ||
1249 | static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); | 1253 | static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); |
1250 | 1254 | ||
1251 | static inline unsigned long rb_page_entries(struct buffer_page *bpage) | 1255 | static void |
1252 | { | 1256 | rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) |
1253 | return local_read(&bpage->entries) & RB_WRITE_MASK; | ||
1254 | } | ||
1255 | |||
1256 | static inline unsigned long rb_page_write(struct buffer_page *bpage) | ||
1257 | { | ||
1258 | return local_read(&bpage->write) & RB_WRITE_MASK; | ||
1259 | } | ||
1260 | |||
1261 | static int | ||
1262 | rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned int nr_pages) | ||
1263 | { | 1257 | { |
1264 | struct list_head *tail_page, *to_remove, *next_page; | 1258 | struct buffer_page *bpage; |
1265 | struct buffer_page *to_remove_page, *tmp_iter_page; | 1259 | struct list_head *p; |
1266 | struct buffer_page *last_page, *first_page; | 1260 | unsigned i; |
1267 | unsigned int nr_removed; | ||
1268 | unsigned long head_bit; | ||
1269 | int page_entries; | ||
1270 | |||
1271 | head_bit = 0; | ||
1272 | |||
1273 | raw_spin_lock_irq(&cpu_buffer->reader_lock); | ||
1274 | atomic_inc(&cpu_buffer->record_disabled); | ||
1275 | /* | ||
1276 | * We don't race with the readers since we have acquired the reader | ||
1277 | * lock. We also don't race with writers after disabling recording. | ||
1278 | * This makes it easy to figure out the first and the last page to be | ||
1279 | * removed from the list. We unlink all the pages in between including | ||
1280 | * the first and last pages. This is done in a busy loop so that we | ||
1281 | * lose the least number of traces. | ||
1282 | * The pages are freed after we restart recording and unlock readers. | ||
1283 | */ | ||
1284 | tail_page = &cpu_buffer->tail_page->list; | ||
1285 | |||
1286 | /* | ||
1287 | * tail page might be on reader page, we remove the next page | ||
1288 | * from the ring buffer | ||
1289 | */ | ||
1290 | if (cpu_buffer->tail_page == cpu_buffer->reader_page) | ||
1291 | tail_page = rb_list_head(tail_page->next); | ||
1292 | to_remove = tail_page; | ||
1293 | 1261 | ||
1294 | /* start of pages to remove */ | 1262 | spin_lock_irq(&cpu_buffer->reader_lock); |
1295 | first_page = list_entry(rb_list_head(to_remove->next), | 1263 | rb_head_page_deactivate(cpu_buffer); |
1296 | struct buffer_page, list); | ||
1297 | 1264 | ||
1298 | for (nr_removed = 0; nr_removed < nr_pages; nr_removed++) { | 1265 | for (i = 0; i < nr_pages; i++) { |
1299 | to_remove = rb_list_head(to_remove)->next; | 1266 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) |
1300 | head_bit |= (unsigned long)to_remove & RB_PAGE_HEAD; | 1267 | goto out; |
1268 | p = cpu_buffer->pages->next; | ||
1269 | bpage = list_entry(p, struct buffer_page, list); | ||
1270 | list_del_init(&bpage->list); | ||
1271 | free_buffer_page(bpage); | ||
1301 | } | 1272 | } |
1273 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) | ||
1274 | goto out; | ||
1302 | 1275 | ||
1303 | next_page = rb_list_head(to_remove)->next; | 1276 | rb_reset_cpu(cpu_buffer); |
1304 | 1277 | rb_check_pages(cpu_buffer); | |
1305 | /* | ||
1306 | * Now we remove all pages between tail_page and next_page. | ||
1307 | * Make sure that we have head_bit value preserved for the | ||
1308 | * next page | ||
1309 | */ | ||
1310 | tail_page->next = (struct list_head *)((unsigned long)next_page | | ||
1311 | head_bit); | ||
1312 | next_page = rb_list_head(next_page); | ||
1313 | next_page->prev = tail_page; | ||
1314 | |||
1315 | /* make sure pages points to a valid page in the ring buffer */ | ||
1316 | cpu_buffer->pages = next_page; | ||
1317 | |||
1318 | /* update head page */ | ||
1319 | if (head_bit) | ||
1320 | cpu_buffer->head_page = list_entry(next_page, | ||
1321 | struct buffer_page, list); | ||
1322 | |||
1323 | /* | ||
1324 | * change read pointer to make sure any read iterators reset | ||
1325 | * themselves | ||
1326 | */ | ||
1327 | cpu_buffer->read = 0; | ||
1328 | |||
1329 | /* pages are removed, resume tracing and then free the pages */ | ||
1330 | atomic_dec(&cpu_buffer->record_disabled); | ||
1331 | raw_spin_unlock_irq(&cpu_buffer->reader_lock); | ||
1332 | |||
1333 | RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)); | ||
1334 | |||
1335 | /* last buffer page to remove */ | ||
1336 | last_page = list_entry(rb_list_head(to_remove), struct buffer_page, | ||
1337 | list); | ||
1338 | tmp_iter_page = first_page; | ||
1339 | |||
1340 | do { | ||
1341 | to_remove_page = tmp_iter_page; | ||
1342 | rb_inc_page(cpu_buffer, &tmp_iter_page); | ||
1343 | |||
1344 | /* update the counters */ | ||
1345 | page_entries = rb_page_entries(to_remove_page); | ||
1346 | if (page_entries) { | ||
1347 | /* | ||
1348 | * If something was added to this page, it was full | ||
1349 | * since it is not the tail page. So we deduct the | ||
1350 | * bytes consumed in ring buffer from here. | ||
1351 | * Increment overrun to account for the lost events. | ||
1352 | */ | ||
1353 | local_add(page_entries, &cpu_buffer->overrun); | ||
1354 | local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes); | ||
1355 | } | ||
1356 | |||
1357 | /* | ||
1358 | * We have already removed references to this list item, just | ||
1359 | * free up the buffer_page and its page | ||
1360 | */ | ||
1361 | free_buffer_page(to_remove_page); | ||
1362 | nr_removed--; | ||
1363 | |||
1364 | } while (to_remove_page != last_page); | ||
1365 | |||
1366 | RB_WARN_ON(cpu_buffer, nr_removed); | ||
1367 | 1278 | ||
1368 | return nr_removed == 0; | 1279 | out: |
1280 | spin_unlock_irq(&cpu_buffer->reader_lock); | ||
1369 | } | 1281 | } |
1370 | 1282 | ||
1371 | static int | 1283 | static void |
1372 | rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer) | 1284 | rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer, |
1285 | struct list_head *pages, unsigned nr_pages) | ||
1373 | { | 1286 | { |
1374 | struct list_head *pages = &cpu_buffer->new_pages; | 1287 | struct buffer_page *bpage; |
1375 | int retries, success; | 1288 | struct list_head *p; |
1376 | 1289 | unsigned i; | |
1377 | raw_spin_lock_irq(&cpu_buffer->reader_lock); | ||
1378 | /* | ||
1379 | * We are holding the reader lock, so the reader page won't be swapped | ||
1380 | * in the ring buffer. Now we are racing with the writer trying to | ||
1381 | * move head page and the tail page. | ||
1382 | * We are going to adapt the reader page update process where: | ||
1383 | * 1. We first splice the start and end of list of new pages between | ||
1384 | * the head page and its previous page. | ||
1385 | * 2. We cmpxchg the prev_page->next to point from head page to the | ||
1386 | * start of new pages list. | ||
1387 | * 3. Finally, we update the head->prev to the end of new list. | ||
1388 | * | ||
1389 | * We will try this process 10 times, to make sure that we don't keep | ||
1390 | * spinning. | ||
1391 | */ | ||
1392 | retries = 10; | ||
1393 | success = 0; | ||
1394 | while (retries--) { | ||
1395 | struct list_head *head_page, *prev_page, *r; | ||
1396 | struct list_head *last_page, *first_page; | ||
1397 | struct list_head *head_page_with_bit; | ||
1398 | |||
1399 | head_page = &rb_set_head_page(cpu_buffer)->list; | ||
1400 | if (!head_page) | ||
1401 | break; | ||
1402 | prev_page = head_page->prev; | ||
1403 | |||
1404 | first_page = pages->next; | ||
1405 | last_page = pages->prev; | ||
1406 | |||
1407 | head_page_with_bit = (struct list_head *) | ||
1408 | ((unsigned long)head_page | RB_PAGE_HEAD); | ||
1409 | |||
1410 | last_page->next = head_page_with_bit; | ||
1411 | first_page->prev = prev_page; | ||
1412 | |||
1413 | r = cmpxchg(&prev_page->next, head_page_with_bit, first_page); | ||
1414 | 1290 | ||
1415 | if (r == head_page_with_bit) { | 1291 | spin_lock_irq(&cpu_buffer->reader_lock); |
1416 | /* | 1292 | rb_head_page_deactivate(cpu_buffer); |
1417 | * yay, we replaced the page pointer to our new list, | ||
1418 | * now, we just have to update to head page's prev | ||
1419 | * pointer to point to end of list | ||
1420 | */ | ||
1421 | head_page->prev = last_page; | ||
1422 | success = 1; | ||
1423 | break; | ||
1424 | } | ||
1425 | } | ||
1426 | 1293 | ||
1427 | if (success) | 1294 | for (i = 0; i < nr_pages; i++) { |
1428 | INIT_LIST_HEAD(pages); | 1295 | if (RB_WARN_ON(cpu_buffer, list_empty(pages))) |
1429 | /* | 1296 | goto out; |
1430 | * If we weren't successful in adding in new pages, warn and stop | 1297 | p = pages->next; |
1431 | * tracing | 1298 | bpage = list_entry(p, struct buffer_page, list); |
1432 | */ | 1299 | list_del_init(&bpage->list); |
1433 | RB_WARN_ON(cpu_buffer, !success); | 1300 | list_add_tail(&bpage->list, cpu_buffer->pages); |
1434 | raw_spin_unlock_irq(&cpu_buffer->reader_lock); | ||
1435 | |||
1436 | /* free pages if they weren't inserted */ | ||
1437 | if (!success) { | ||
1438 | struct buffer_page *bpage, *tmp; | ||
1439 | list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, | ||
1440 | list) { | ||
1441 | list_del_init(&bpage->list); | ||
1442 | free_buffer_page(bpage); | ||
1443 | } | ||
1444 | } | 1301 | } |
1445 | return success; | 1302 | rb_reset_cpu(cpu_buffer); |
1446 | } | 1303 | rb_check_pages(cpu_buffer); |
1447 | |||
1448 | static void rb_update_pages(struct ring_buffer_per_cpu *cpu_buffer) | ||
1449 | { | ||
1450 | int success; | ||
1451 | |||
1452 | if (cpu_buffer->nr_pages_to_update > 0) | ||
1453 | success = rb_insert_pages(cpu_buffer); | ||
1454 | else | ||
1455 | success = rb_remove_pages(cpu_buffer, | ||
1456 | -cpu_buffer->nr_pages_to_update); | ||
1457 | |||
1458 | if (success) | ||
1459 | cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update; | ||
1460 | } | ||
1461 | 1304 | ||
1462 | static void update_pages_handler(struct work_struct *work) | 1305 | out: |
1463 | { | 1306 | spin_unlock_irq(&cpu_buffer->reader_lock); |
1464 | struct ring_buffer_per_cpu *cpu_buffer = container_of(work, | ||
1465 | struct ring_buffer_per_cpu, update_pages_work); | ||
1466 | rb_update_pages(cpu_buffer); | ||
1467 | complete(&cpu_buffer->update_done); | ||
1468 | } | 1307 | } |
1469 | 1308 | ||
1470 | /** | 1309 | /** |
@@ -1474,14 +1313,16 @@ static void update_pages_handler(struct work_struct *work) | |||
1474 | * | 1313 | * |
1475 | * Minimum size is 2 * BUF_PAGE_SIZE. | 1314 | * Minimum size is 2 * BUF_PAGE_SIZE. |
1476 | * | 1315 | * |
1477 | * Returns 0 on success and < 0 on failure. | 1316 | * Returns -1 on failure. |
1478 | */ | 1317 | */ |
1479 | int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size, | 1318 | int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) |
1480 | int cpu_id) | ||
1481 | { | 1319 | { |
1482 | struct ring_buffer_per_cpu *cpu_buffer; | 1320 | struct ring_buffer_per_cpu *cpu_buffer; |
1483 | unsigned nr_pages; | 1321 | unsigned nr_pages, rm_pages, new_pages; |
1484 | int cpu, err = 0; | 1322 | struct buffer_page *bpage, *tmp; |
1323 | unsigned long buffer_size; | ||
1324 | LIST_HEAD(pages); | ||
1325 | int i, cpu; | ||
1485 | 1326 | ||
1486 | /* | 1327 | /* |
1487 | * Always succeed at resizing a non-existent buffer: | 1328 | * Always succeed at resizing a non-existent buffer: |
@@ -1489,165 +1330,115 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size, | |||
1489 | if (!buffer) | 1330 | if (!buffer) |
1490 | return size; | 1331 | return size; |
1491 | 1332 | ||
1492 | /* Make sure the requested buffer exists */ | ||
1493 | if (cpu_id != RING_BUFFER_ALL_CPUS && | ||
1494 | !cpumask_test_cpu(cpu_id, buffer->cpumask)) | ||
1495 | return size; | ||
1496 | |||
1497 | size = DIV_ROUND_UP(size, BUF_PAGE_SIZE); | 1333 | size = DIV_ROUND_UP(size, BUF_PAGE_SIZE); |
1498 | size *= BUF_PAGE_SIZE; | 1334 | size *= BUF_PAGE_SIZE; |
1335 | buffer_size = buffer->pages * BUF_PAGE_SIZE; | ||
1499 | 1336 | ||
1500 | /* we need a minimum of two pages */ | 1337 | /* we need a minimum of two pages */ |
1501 | if (size < BUF_PAGE_SIZE * 2) | 1338 | if (size < BUF_PAGE_SIZE * 2) |
1502 | size = BUF_PAGE_SIZE * 2; | 1339 | size = BUF_PAGE_SIZE * 2; |
1503 | 1340 | ||
1504 | nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); | 1341 | if (size == buffer_size) |
1342 | return size; | ||
1505 | 1343 | ||
1506 | /* | 1344 | atomic_inc(&buffer->record_disabled); |
1507 | * Don't succeed if resizing is disabled, as a reader might be | 1345 | |
1508 | * manipulating the ring buffer and is expecting a sane state while | 1346 | /* Make sure all writers are done with this buffer. */ |
1509 | * this is true. | 1347 | synchronize_sched(); |
1510 | */ | ||
1511 | if (atomic_read(&buffer->resize_disabled)) | ||
1512 | return -EBUSY; | ||
1513 | 1348 | ||
1514 | /* prevent another thread from changing buffer sizes */ | ||
1515 | mutex_lock(&buffer->mutex); | 1349 | mutex_lock(&buffer->mutex); |
1350 | get_online_cpus(); | ||
1516 | 1351 | ||
1517 | if (cpu_id == RING_BUFFER_ALL_CPUS) { | 1352 | nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); |
1518 | /* calculate the pages to update */ | ||
1519 | for_each_buffer_cpu(buffer, cpu) { | ||
1520 | cpu_buffer = buffer->buffers[cpu]; | ||
1521 | 1353 | ||
1522 | cpu_buffer->nr_pages_to_update = nr_pages - | 1354 | if (size < buffer_size) { |
1523 | cpu_buffer->nr_pages; | ||
1524 | /* | ||
1525 | * nothing more to do for removing pages or no update | ||
1526 | */ | ||
1527 | if (cpu_buffer->nr_pages_to_update <= 0) | ||
1528 | continue; | ||
1529 | /* | ||
1530 | * to add pages, make sure all new pages can be | ||
1531 | * allocated without receiving ENOMEM | ||
1532 | */ | ||
1533 | INIT_LIST_HEAD(&cpu_buffer->new_pages); | ||
1534 | if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update, | ||
1535 | &cpu_buffer->new_pages, cpu)) { | ||
1536 | /* not enough memory for new pages */ | ||
1537 | err = -ENOMEM; | ||
1538 | goto out_err; | ||
1539 | } | ||
1540 | } | ||
1541 | 1355 | ||
1542 | get_online_cpus(); | 1356 | /* easy case, just free pages */ |
1543 | /* | 1357 | if (RB_WARN_ON(buffer, nr_pages >= buffer->pages)) |
1544 | * Fire off all the required work handlers | 1358 | goto out_fail; |
1545 | * We can't schedule on offline CPUs, but it's not necessary | 1359 | |
1546 | * since we can change their buffer sizes without any race. | 1360 | rm_pages = buffer->pages - nr_pages; |
1547 | */ | ||
1548 | for_each_buffer_cpu(buffer, cpu) { | ||
1549 | cpu_buffer = buffer->buffers[cpu]; | ||
1550 | if (!cpu_buffer->nr_pages_to_update) | ||
1551 | continue; | ||
1552 | |||
1553 | if (cpu_online(cpu)) | ||
1554 | schedule_work_on(cpu, | ||
1555 | &cpu_buffer->update_pages_work); | ||
1556 | else | ||
1557 | rb_update_pages(cpu_buffer); | ||
1558 | } | ||
1559 | 1361 | ||
1560 | /* wait for all the updates to complete */ | ||
1561 | for_each_buffer_cpu(buffer, cpu) { | 1362 | for_each_buffer_cpu(buffer, cpu) { |
1562 | cpu_buffer = buffer->buffers[cpu]; | 1363 | cpu_buffer = buffer->buffers[cpu]; |
1563 | if (!cpu_buffer->nr_pages_to_update) | 1364 | rb_remove_pages(cpu_buffer, rm_pages); |
1564 | continue; | ||
1565 | |||
1566 | if (cpu_online(cpu)) | ||
1567 | wait_for_completion(&cpu_buffer->update_done); | ||
1568 | cpu_buffer->nr_pages_to_update = 0; | ||
1569 | } | 1365 | } |
1366 | goto out; | ||
1367 | } | ||
1570 | 1368 | ||
1571 | put_online_cpus(); | 1369 | /* |
1572 | } else { | 1370 | * This is a bit more difficult. We only want to add pages |
1573 | /* Make sure this CPU has been intitialized */ | 1371 | * when we can allocate enough for all CPUs. We do this |
1574 | if (!cpumask_test_cpu(cpu_id, buffer->cpumask)) | 1372 | * by allocating all the pages and storing them on a local |
1575 | goto out; | 1373 | * link list. If we succeed in our allocation, then we |
1576 | 1374 | * add these pages to the cpu_buffers. Otherwise we just free | |
1577 | cpu_buffer = buffer->buffers[cpu_id]; | 1375 | * them all and return -ENOMEM; |
1578 | 1376 | */ | |
1579 | if (nr_pages == cpu_buffer->nr_pages) | 1377 | if (RB_WARN_ON(buffer, nr_pages <= buffer->pages)) |
1580 | goto out; | 1378 | goto out_fail; |
1581 | 1379 | ||
1582 | cpu_buffer->nr_pages_to_update = nr_pages - | 1380 | new_pages = nr_pages - buffer->pages; |
1583 | cpu_buffer->nr_pages; | ||
1584 | 1381 | ||
1585 | INIT_LIST_HEAD(&cpu_buffer->new_pages); | 1382 | for_each_buffer_cpu(buffer, cpu) { |
1586 | if (cpu_buffer->nr_pages_to_update > 0 && | 1383 | for (i = 0; i < new_pages; i++) { |
1587 | __rb_allocate_pages(cpu_buffer->nr_pages_to_update, | 1384 | struct page *page; |
1588 | &cpu_buffer->new_pages, cpu_id)) { | 1385 | /* |
1589 | err = -ENOMEM; | 1386 | * __GFP_NORETRY flag makes sure that the allocation |
1590 | goto out_err; | 1387 | * fails gracefully without invoking oom-killer and |
1388 | * the system is not destabilized. | ||
1389 | */ | ||
1390 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), | ||
1391 | cache_line_size()), | ||
1392 | GFP_KERNEL | __GFP_NORETRY, | ||
1393 | cpu_to_node(cpu)); | ||
1394 | if (!bpage) | ||
1395 | goto free_pages; | ||
1396 | list_add(&bpage->list, &pages); | ||
1397 | page = alloc_pages_node(cpu_to_node(cpu), | ||
1398 | GFP_KERNEL | __GFP_NORETRY, 0); | ||
1399 | if (!page) | ||
1400 | goto free_pages; | ||
1401 | bpage->page = page_address(page); | ||
1402 | rb_init_page(bpage->page); | ||
1591 | } | 1403 | } |
1592 | |||
1593 | get_online_cpus(); | ||
1594 | |||
1595 | if (cpu_online(cpu_id)) { | ||
1596 | schedule_work_on(cpu_id, | ||
1597 | &cpu_buffer->update_pages_work); | ||
1598 | wait_for_completion(&cpu_buffer->update_done); | ||
1599 | } else | ||
1600 | rb_update_pages(cpu_buffer); | ||
1601 | |||
1602 | cpu_buffer->nr_pages_to_update = 0; | ||
1603 | put_online_cpus(); | ||
1604 | } | 1404 | } |
1605 | 1405 | ||
1606 | out: | 1406 | for_each_buffer_cpu(buffer, cpu) { |
1607 | /* | 1407 | cpu_buffer = buffer->buffers[cpu]; |
1608 | * The ring buffer resize can happen with the ring buffer | 1408 | rb_insert_pages(cpu_buffer, &pages, new_pages); |
1609 | * enabled, so that the update disturbs the tracing as little | ||
1610 | * as possible. But if the buffer is disabled, we do not need | ||
1611 | * to worry about that, and we can take the time to verify | ||
1612 | * that the buffer is not corrupt. | ||
1613 | */ | ||
1614 | if (atomic_read(&buffer->record_disabled)) { | ||
1615 | atomic_inc(&buffer->record_disabled); | ||
1616 | /* | ||
1617 | * Even though the buffer was disabled, we must make sure | ||
1618 | * that it is truly disabled before calling rb_check_pages. | ||
1619 | * There could have been a race between checking | ||
1620 | * record_disable and incrementing it. | ||
1621 | */ | ||
1622 | synchronize_sched(); | ||
1623 | for_each_buffer_cpu(buffer, cpu) { | ||
1624 | cpu_buffer = buffer->buffers[cpu]; | ||
1625 | rb_check_pages(cpu_buffer); | ||
1626 | } | ||
1627 | atomic_dec(&buffer->record_disabled); | ||
1628 | } | 1409 | } |
1629 | 1410 | ||
1630 | mutex_unlock(&buffer->mutex); | 1411 | if (RB_WARN_ON(buffer, !list_empty(&pages))) |
1631 | return size; | 1412 | goto out_fail; |
1632 | 1413 | ||
1633 | out_err: | 1414 | out: |
1634 | for_each_buffer_cpu(buffer, cpu) { | 1415 | buffer->pages = nr_pages; |
1635 | struct buffer_page *bpage, *tmp; | 1416 | put_online_cpus(); |
1417 | mutex_unlock(&buffer->mutex); | ||
1636 | 1418 | ||
1637 | cpu_buffer = buffer->buffers[cpu]; | 1419 | atomic_dec(&buffer->record_disabled); |
1638 | cpu_buffer->nr_pages_to_update = 0; | ||
1639 | 1420 | ||
1640 | if (list_empty(&cpu_buffer->new_pages)) | 1421 | return size; |
1641 | continue; | ||
1642 | 1422 | ||
1643 | list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, | 1423 | free_pages: |
1644 | list) { | 1424 | list_for_each_entry_safe(bpage, tmp, &pages, list) { |
1645 | list_del_init(&bpage->list); | 1425 | list_del_init(&bpage->list); |
1646 | free_buffer_page(bpage); | 1426 | free_buffer_page(bpage); |
1647 | } | ||
1648 | } | 1427 | } |
1428 | put_online_cpus(); | ||
1649 | mutex_unlock(&buffer->mutex); | 1429 | mutex_unlock(&buffer->mutex); |
1650 | return err; | 1430 | atomic_dec(&buffer->record_disabled); |
1431 | return -ENOMEM; | ||
1432 | |||
1433 | /* | ||
1434 | * Something went totally wrong, and we are too paranoid | ||
1435 | * to even clean up the mess. | ||
1436 | */ | ||
1437 | out_fail: | ||
1438 | put_online_cpus(); | ||
1439 | mutex_unlock(&buffer->mutex); | ||
1440 | atomic_dec(&buffer->record_disabled); | ||
1441 | return -1; | ||
1651 | } | 1442 | } |
1652 | EXPORT_SYMBOL_GPL(ring_buffer_resize); | 1443 | EXPORT_SYMBOL_GPL(ring_buffer_resize); |
1653 | 1444 | ||
@@ -1686,11 +1477,21 @@ rb_iter_head_event(struct ring_buffer_iter *iter) | |||
1686 | return __rb_page_index(iter->head_page, iter->head); | 1477 | return __rb_page_index(iter->head_page, iter->head); |
1687 | } | 1478 | } |
1688 | 1479 | ||
1480 | static inline unsigned long rb_page_write(struct buffer_page *bpage) | ||
1481 | { | ||
1482 | return local_read(&bpage->write) & RB_WRITE_MASK; | ||
1483 | } | ||
1484 | |||
1689 | static inline unsigned rb_page_commit(struct buffer_page *bpage) | 1485 | static inline unsigned rb_page_commit(struct buffer_page *bpage) |
1690 | { | 1486 | { |
1691 | return local_read(&bpage->page->commit); | 1487 | return local_read(&bpage->page->commit); |
1692 | } | 1488 | } |
1693 | 1489 | ||
1490 | static inline unsigned long rb_page_entries(struct buffer_page *bpage) | ||
1491 | { | ||
1492 | return local_read(&bpage->entries) & RB_WRITE_MASK; | ||
1493 | } | ||
1494 | |||
1694 | /* Size is determined by what has been committed */ | 1495 | /* Size is determined by what has been committed */ |
1695 | static inline unsigned rb_page_size(struct buffer_page *bpage) | 1496 | static inline unsigned rb_page_size(struct buffer_page *bpage) |
1696 | { | 1497 | { |
@@ -1739,7 +1540,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) | |||
1739 | * assign the commit to the tail. | 1540 | * assign the commit to the tail. |
1740 | */ | 1541 | */ |
1741 | again: | 1542 | again: |
1742 | max_count = cpu_buffer->nr_pages * 100; | 1543 | max_count = cpu_buffer->buffer->pages * 100; |
1743 | 1544 | ||
1744 | while (cpu_buffer->commit_page != cpu_buffer->tail_page) { | 1545 | while (cpu_buffer->commit_page != cpu_buffer->tail_page) { |
1745 | if (RB_WARN_ON(cpu_buffer, !(--max_count))) | 1546 | if (RB_WARN_ON(cpu_buffer, !(--max_count))) |
@@ -1823,7 +1624,7 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta) | |||
1823 | } | 1624 | } |
1824 | 1625 | ||
1825 | /** | 1626 | /** |
1826 | * rb_update_event - update event type and data | 1627 | * ring_buffer_update_event - update event type and data |
1827 | * @event: the even to update | 1628 | * @event: the even to update |
1828 | * @type: the type of event | 1629 | * @type: the type of event |
1829 | * @length: the size of the event field in the ring buffer | 1630 | * @length: the size of the event field in the ring buffer |
@@ -1907,7 +1708,6 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer, | |||
1907 | * the counters. | 1708 | * the counters. |
1908 | */ | 1709 | */ |
1909 | local_add(entries, &cpu_buffer->overrun); | 1710 | local_add(entries, &cpu_buffer->overrun); |
1910 | local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes); | ||
1911 | 1711 | ||
1912 | /* | 1712 | /* |
1913 | * The entries will be zeroed out when we move the | 1713 | * The entries will be zeroed out when we move the |
@@ -2063,9 +1863,6 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, | |||
2063 | event = __rb_page_index(tail_page, tail); | 1863 | event = __rb_page_index(tail_page, tail); |
2064 | kmemcheck_annotate_bitfield(event, bitfield); | 1864 | kmemcheck_annotate_bitfield(event, bitfield); |
2065 | 1865 | ||
2066 | /* account for padding bytes */ | ||
2067 | local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes); | ||
2068 | |||
2069 | /* | 1866 | /* |
2070 | * Save the original length to the meta data. | 1867 | * Save the original length to the meta data. |
2071 | * This will be used by the reader to add lost event | 1868 | * This will be used by the reader to add lost event |
@@ -2158,10 +1955,8 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer, | |||
2158 | * If we are not in overwrite mode, | 1955 | * If we are not in overwrite mode, |
2159 | * this is easy, just stop here. | 1956 | * this is easy, just stop here. |
2160 | */ | 1957 | */ |
2161 | if (!(buffer->flags & RB_FL_OVERWRITE)) { | 1958 | if (!(buffer->flags & RB_FL_OVERWRITE)) |
2162 | local_inc(&cpu_buffer->dropped_events); | ||
2163 | goto out_reset; | 1959 | goto out_reset; |
2164 | } | ||
2165 | 1960 | ||
2166 | ret = rb_handle_head_page(cpu_buffer, | 1961 | ret = rb_handle_head_page(cpu_buffer, |
2167 | tail_page, | 1962 | tail_page, |
@@ -2259,9 +2054,6 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, | |||
2259 | if (!tail) | 2054 | if (!tail) |
2260 | tail_page->page->time_stamp = ts; | 2055 | tail_page->page->time_stamp = ts; |
2261 | 2056 | ||
2262 | /* account for these added bytes */ | ||
2263 | local_add(length, &cpu_buffer->entries_bytes); | ||
2264 | |||
2265 | return event; | 2057 | return event; |
2266 | } | 2058 | } |
2267 | 2059 | ||
@@ -2284,7 +2076,6 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, | |||
2284 | if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) { | 2076 | if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) { |
2285 | unsigned long write_mask = | 2077 | unsigned long write_mask = |
2286 | local_read(&bpage->write) & ~RB_WRITE_MASK; | 2078 | local_read(&bpage->write) & ~RB_WRITE_MASK; |
2287 | unsigned long event_length = rb_event_length(event); | ||
2288 | /* | 2079 | /* |
2289 | * This is on the tail page. It is possible that | 2080 | * This is on the tail page. It is possible that |
2290 | * a write could come in and move the tail page | 2081 | * a write could come in and move the tail page |
@@ -2294,11 +2085,8 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, | |||
2294 | old_index += write_mask; | 2085 | old_index += write_mask; |
2295 | new_index += write_mask; | 2086 | new_index += write_mask; |
2296 | index = local_cmpxchg(&bpage->write, old_index, new_index); | 2087 | index = local_cmpxchg(&bpage->write, old_index, new_index); |
2297 | if (index == old_index) { | 2088 | if (index == old_index) |
2298 | /* update counters */ | ||
2299 | local_sub(event_length, &cpu_buffer->entries_bytes); | ||
2300 | return 1; | 2089 | return 1; |
2301 | } | ||
2302 | } | 2090 | } |
2303 | 2091 | ||
2304 | /* could not discard */ | 2092 | /* could not discard */ |
@@ -2725,8 +2513,8 @@ EXPORT_SYMBOL_GPL(ring_buffer_discard_commit); | |||
2725 | * and not the length of the event which would hold the header. | 2513 | * and not the length of the event which would hold the header. |
2726 | */ | 2514 | */ |
2727 | int ring_buffer_write(struct ring_buffer *buffer, | 2515 | int ring_buffer_write(struct ring_buffer *buffer, |
2728 | unsigned long length, | 2516 | unsigned long length, |
2729 | void *data) | 2517 | void *data) |
2730 | { | 2518 | { |
2731 | struct ring_buffer_per_cpu *cpu_buffer; | 2519 | struct ring_buffer_per_cpu *cpu_buffer; |
2732 | struct ring_buffer_event *event; | 2520 | struct ring_buffer_event *event; |
@@ -2818,63 +2606,6 @@ void ring_buffer_record_enable(struct ring_buffer *buffer) | |||
2818 | EXPORT_SYMBOL_GPL(ring_buffer_record_enable); | 2606 | EXPORT_SYMBOL_GPL(ring_buffer_record_enable); |
2819 | 2607 | ||
2820 | /** | 2608 | /** |
2821 | * ring_buffer_record_off - stop all writes into the buffer | ||
2822 | * @buffer: The ring buffer to stop writes to. | ||
2823 | * | ||
2824 | * This prevents all writes to the buffer. Any attempt to write | ||
2825 | * to the buffer after this will fail and return NULL. | ||
2826 | * | ||
2827 | * This is different than ring_buffer_record_disable() as | ||
2828 | * it works like an on/off switch, where as the disable() version | ||
2829 | * must be paired with a enable(). | ||
2830 | */ | ||
2831 | void ring_buffer_record_off(struct ring_buffer *buffer) | ||
2832 | { | ||
2833 | unsigned int rd; | ||
2834 | unsigned int new_rd; | ||
2835 | |||
2836 | do { | ||
2837 | rd = atomic_read(&buffer->record_disabled); | ||
2838 | new_rd = rd | RB_BUFFER_OFF; | ||
2839 | } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd); | ||
2840 | } | ||
2841 | EXPORT_SYMBOL_GPL(ring_buffer_record_off); | ||
2842 | |||
2843 | /** | ||
2844 | * ring_buffer_record_on - restart writes into the buffer | ||
2845 | * @buffer: The ring buffer to start writes to. | ||
2846 | * | ||
2847 | * This enables all writes to the buffer that was disabled by | ||
2848 | * ring_buffer_record_off(). | ||
2849 | * | ||
2850 | * This is different than ring_buffer_record_enable() as | ||
2851 | * it works like an on/off switch, where as the enable() version | ||
2852 | * must be paired with a disable(). | ||
2853 | */ | ||
2854 | void ring_buffer_record_on(struct ring_buffer *buffer) | ||
2855 | { | ||
2856 | unsigned int rd; | ||
2857 | unsigned int new_rd; | ||
2858 | |||
2859 | do { | ||
2860 | rd = atomic_read(&buffer->record_disabled); | ||
2861 | new_rd = rd & ~RB_BUFFER_OFF; | ||
2862 | } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd); | ||
2863 | } | ||
2864 | EXPORT_SYMBOL_GPL(ring_buffer_record_on); | ||
2865 | |||
2866 | /** | ||
2867 | * ring_buffer_record_is_on - return true if the ring buffer can write | ||
2868 | * @buffer: The ring buffer to see if write is enabled | ||
2869 | * | ||
2870 | * Returns true if the ring buffer is in a state that it accepts writes. | ||
2871 | */ | ||
2872 | int ring_buffer_record_is_on(struct ring_buffer *buffer) | ||
2873 | { | ||
2874 | return !atomic_read(&buffer->record_disabled); | ||
2875 | } | ||
2876 | |||
2877 | /** | ||
2878 | * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer | 2609 | * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer |
2879 | * @buffer: The ring buffer to stop writes to. | 2610 | * @buffer: The ring buffer to stop writes to. |
2880 | * @cpu: The CPU buffer to stop | 2611 | * @cpu: The CPU buffer to stop |
@@ -2930,59 +2661,6 @@ rb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer) | |||
2930 | } | 2661 | } |
2931 | 2662 | ||
2932 | /** | 2663 | /** |
2933 | * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer | ||
2934 | * @buffer: The ring buffer | ||
2935 | * @cpu: The per CPU buffer to read from. | ||
2936 | */ | ||
2937 | u64 ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu) | ||
2938 | { | ||
2939 | unsigned long flags; | ||
2940 | struct ring_buffer_per_cpu *cpu_buffer; | ||
2941 | struct buffer_page *bpage; | ||
2942 | u64 ret = 0; | ||
2943 | |||
2944 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | ||
2945 | return 0; | ||
2946 | |||
2947 | cpu_buffer = buffer->buffers[cpu]; | ||
2948 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | ||
2949 | /* | ||
2950 | * if the tail is on reader_page, oldest time stamp is on the reader | ||
2951 | * page | ||
2952 | */ | ||
2953 | if (cpu_buffer->tail_page == cpu_buffer->reader_page) | ||
2954 | bpage = cpu_buffer->reader_page; | ||
2955 | else | ||
2956 | bpage = rb_set_head_page(cpu_buffer); | ||
2957 | if (bpage) | ||
2958 | ret = bpage->page->time_stamp; | ||
2959 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | ||
2960 | |||
2961 | return ret; | ||
2962 | } | ||
2963 | EXPORT_SYMBOL_GPL(ring_buffer_oldest_event_ts); | ||
2964 | |||
2965 | /** | ||
2966 | * ring_buffer_bytes_cpu - get the number of bytes consumed in a cpu buffer | ||
2967 | * @buffer: The ring buffer | ||
2968 | * @cpu: The per CPU buffer to read from. | ||
2969 | */ | ||
2970 | unsigned long ring_buffer_bytes_cpu(struct ring_buffer *buffer, int cpu) | ||
2971 | { | ||
2972 | struct ring_buffer_per_cpu *cpu_buffer; | ||
2973 | unsigned long ret; | ||
2974 | |||
2975 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | ||
2976 | return 0; | ||
2977 | |||
2978 | cpu_buffer = buffer->buffers[cpu]; | ||
2979 | ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes; | ||
2980 | |||
2981 | return ret; | ||
2982 | } | ||
2983 | EXPORT_SYMBOL_GPL(ring_buffer_bytes_cpu); | ||
2984 | |||
2985 | /** | ||
2986 | * ring_buffer_entries_cpu - get the number of entries in a cpu buffer | 2664 | * ring_buffer_entries_cpu - get the number of entries in a cpu buffer |
2987 | * @buffer: The ring buffer | 2665 | * @buffer: The ring buffer |
2988 | * @cpu: The per CPU buffer to get the entries from. | 2666 | * @cpu: The per CPU buffer to get the entries from. |
@@ -3001,8 +2679,7 @@ unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu) | |||
3001 | EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu); | 2679 | EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu); |
3002 | 2680 | ||
3003 | /** | 2681 | /** |
3004 | * ring_buffer_overrun_cpu - get the number of overruns caused by the ring | 2682 | * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer |
3005 | * buffer wrapping around (only if RB_FL_OVERWRITE is on). | ||
3006 | * @buffer: The ring buffer | 2683 | * @buffer: The ring buffer |
3007 | * @cpu: The per CPU buffer to get the number of overruns from | 2684 | * @cpu: The per CPU buffer to get the number of overruns from |
3008 | */ | 2685 | */ |
@@ -3022,9 +2699,7 @@ unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu) | |||
3022 | EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu); | 2699 | EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu); |
3023 | 2700 | ||
3024 | /** | 2701 | /** |
3025 | * ring_buffer_commit_overrun_cpu - get the number of overruns caused by | 2702 | * ring_buffer_commit_overrun_cpu - get the number of overruns caused by commits |
3026 | * commits failing due to the buffer wrapping around while there are uncommitted | ||
3027 | * events, such as during an interrupt storm. | ||
3028 | * @buffer: The ring buffer | 2703 | * @buffer: The ring buffer |
3029 | * @cpu: The per CPU buffer to get the number of overruns from | 2704 | * @cpu: The per CPU buffer to get the number of overruns from |
3030 | */ | 2705 | */ |
@@ -3045,28 +2720,6 @@ ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu) | |||
3045 | EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu); | 2720 | EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu); |
3046 | 2721 | ||
3047 | /** | 2722 | /** |
3048 | * ring_buffer_dropped_events_cpu - get the number of dropped events caused by | ||
3049 | * the ring buffer filling up (only if RB_FL_OVERWRITE is off). | ||
3050 | * @buffer: The ring buffer | ||
3051 | * @cpu: The per CPU buffer to get the number of overruns from | ||
3052 | */ | ||
3053 | unsigned long | ||
3054 | ring_buffer_dropped_events_cpu(struct ring_buffer *buffer, int cpu) | ||
3055 | { | ||
3056 | struct ring_buffer_per_cpu *cpu_buffer; | ||
3057 | unsigned long ret; | ||
3058 | |||
3059 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | ||
3060 | return 0; | ||
3061 | |||
3062 | cpu_buffer = buffer->buffers[cpu]; | ||
3063 | ret = local_read(&cpu_buffer->dropped_events); | ||
3064 | |||
3065 | return ret; | ||
3066 | } | ||
3067 | EXPORT_SYMBOL_GPL(ring_buffer_dropped_events_cpu); | ||
3068 | |||
3069 | /** | ||
3070 | * ring_buffer_entries - get the number of entries in a buffer | 2723 | * ring_buffer_entries - get the number of entries in a buffer |
3071 | * @buffer: The ring buffer | 2724 | * @buffer: The ring buffer |
3072 | * | 2725 | * |
@@ -3151,9 +2804,9 @@ void ring_buffer_iter_reset(struct ring_buffer_iter *iter) | |||
3151 | 2804 | ||
3152 | cpu_buffer = iter->cpu_buffer; | 2805 | cpu_buffer = iter->cpu_buffer; |
3153 | 2806 | ||
3154 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | 2807 | spin_lock_irqsave(&cpu_buffer->reader_lock, flags); |
3155 | rb_iter_reset(iter); | 2808 | rb_iter_reset(iter); |
3156 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | 2809 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); |
3157 | } | 2810 | } |
3158 | EXPORT_SYMBOL_GPL(ring_buffer_iter_reset); | 2811 | EXPORT_SYMBOL_GPL(ring_buffer_iter_reset); |
3159 | 2812 | ||
@@ -3274,10 +2927,6 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) | |||
3274 | if (cpu_buffer->commit_page == cpu_buffer->reader_page) | 2927 | if (cpu_buffer->commit_page == cpu_buffer->reader_page) |
3275 | goto out; | 2928 | goto out; |
3276 | 2929 | ||
3277 | /* Don't bother swapping if the ring buffer is empty */ | ||
3278 | if (rb_num_of_entries(cpu_buffer) == 0) | ||
3279 | goto out; | ||
3280 | |||
3281 | /* | 2930 | /* |
3282 | * Reset the reader page to size zero. | 2931 | * Reset the reader page to size zero. |
3283 | */ | 2932 | */ |
@@ -3291,8 +2940,6 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) | |||
3291 | * Splice the empty reader page into the list around the head. | 2940 | * Splice the empty reader page into the list around the head. |
3292 | */ | 2941 | */ |
3293 | reader = rb_set_head_page(cpu_buffer); | 2942 | reader = rb_set_head_page(cpu_buffer); |
3294 | if (!reader) | ||
3295 | goto out; | ||
3296 | cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next); | 2943 | cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next); |
3297 | cpu_buffer->reader_page->list.prev = reader->list.prev; | 2944 | cpu_buffer->reader_page->list.prev = reader->list.prev; |
3298 | 2945 | ||
@@ -3618,12 +3265,12 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts, | |||
3618 | again: | 3265 | again: |
3619 | local_irq_save(flags); | 3266 | local_irq_save(flags); |
3620 | if (dolock) | 3267 | if (dolock) |
3621 | raw_spin_lock(&cpu_buffer->reader_lock); | 3268 | spin_lock(&cpu_buffer->reader_lock); |
3622 | event = rb_buffer_peek(cpu_buffer, ts, lost_events); | 3269 | event = rb_buffer_peek(cpu_buffer, ts, lost_events); |
3623 | if (event && event->type_len == RINGBUF_TYPE_PADDING) | 3270 | if (event && event->type_len == RINGBUF_TYPE_PADDING) |
3624 | rb_advance_reader(cpu_buffer); | 3271 | rb_advance_reader(cpu_buffer); |
3625 | if (dolock) | 3272 | if (dolock) |
3626 | raw_spin_unlock(&cpu_buffer->reader_lock); | 3273 | spin_unlock(&cpu_buffer->reader_lock); |
3627 | local_irq_restore(flags); | 3274 | local_irq_restore(flags); |
3628 | 3275 | ||
3629 | if (event && event->type_len == RINGBUF_TYPE_PADDING) | 3276 | if (event && event->type_len == RINGBUF_TYPE_PADDING) |
@@ -3648,9 +3295,9 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts) | |||
3648 | unsigned long flags; | 3295 | unsigned long flags; |
3649 | 3296 | ||
3650 | again: | 3297 | again: |
3651 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | 3298 | spin_lock_irqsave(&cpu_buffer->reader_lock, flags); |
3652 | event = rb_iter_peek(iter, ts); | 3299 | event = rb_iter_peek(iter, ts); |
3653 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | 3300 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); |
3654 | 3301 | ||
3655 | if (event && event->type_len == RINGBUF_TYPE_PADDING) | 3302 | if (event && event->type_len == RINGBUF_TYPE_PADDING) |
3656 | goto again; | 3303 | goto again; |
@@ -3690,7 +3337,7 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts, | |||
3690 | cpu_buffer = buffer->buffers[cpu]; | 3337 | cpu_buffer = buffer->buffers[cpu]; |
3691 | local_irq_save(flags); | 3338 | local_irq_save(flags); |
3692 | if (dolock) | 3339 | if (dolock) |
3693 | raw_spin_lock(&cpu_buffer->reader_lock); | 3340 | spin_lock(&cpu_buffer->reader_lock); |
3694 | 3341 | ||
3695 | event = rb_buffer_peek(cpu_buffer, ts, lost_events); | 3342 | event = rb_buffer_peek(cpu_buffer, ts, lost_events); |
3696 | if (event) { | 3343 | if (event) { |
@@ -3699,7 +3346,7 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts, | |||
3699 | } | 3346 | } |
3700 | 3347 | ||
3701 | if (dolock) | 3348 | if (dolock) |
3702 | raw_spin_unlock(&cpu_buffer->reader_lock); | 3349 | spin_unlock(&cpu_buffer->reader_lock); |
3703 | local_irq_restore(flags); | 3350 | local_irq_restore(flags); |
3704 | 3351 | ||
3705 | out: | 3352 | out: |
@@ -3749,7 +3396,6 @@ ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu) | |||
3749 | 3396 | ||
3750 | iter->cpu_buffer = cpu_buffer; | 3397 | iter->cpu_buffer = cpu_buffer; |
3751 | 3398 | ||
3752 | atomic_inc(&buffer->resize_disabled); | ||
3753 | atomic_inc(&cpu_buffer->record_disabled); | 3399 | atomic_inc(&cpu_buffer->record_disabled); |
3754 | 3400 | ||
3755 | return iter; | 3401 | return iter; |
@@ -3792,11 +3438,11 @@ ring_buffer_read_start(struct ring_buffer_iter *iter) | |||
3792 | 3438 | ||
3793 | cpu_buffer = iter->cpu_buffer; | 3439 | cpu_buffer = iter->cpu_buffer; |
3794 | 3440 | ||
3795 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | 3441 | spin_lock_irqsave(&cpu_buffer->reader_lock, flags); |
3796 | arch_spin_lock(&cpu_buffer->lock); | 3442 | arch_spin_lock(&cpu_buffer->lock); |
3797 | rb_iter_reset(iter); | 3443 | rb_iter_reset(iter); |
3798 | arch_spin_unlock(&cpu_buffer->lock); | 3444 | arch_spin_unlock(&cpu_buffer->lock); |
3799 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | 3445 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); |
3800 | } | 3446 | } |
3801 | EXPORT_SYMBOL_GPL(ring_buffer_read_start); | 3447 | EXPORT_SYMBOL_GPL(ring_buffer_read_start); |
3802 | 3448 | ||
@@ -3811,20 +3457,8 @@ void | |||
3811 | ring_buffer_read_finish(struct ring_buffer_iter *iter) | 3457 | ring_buffer_read_finish(struct ring_buffer_iter *iter) |
3812 | { | 3458 | { |
3813 | struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; | 3459 | struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; |
3814 | unsigned long flags; | ||
3815 | |||
3816 | /* | ||
3817 | * Ring buffer is disabled from recording, here's a good place | ||
3818 | * to check the integrity of the ring buffer. | ||
3819 | * Must prevent readers from trying to read, as the check | ||
3820 | * clears the HEAD page and readers require it. | ||
3821 | */ | ||
3822 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | ||
3823 | rb_check_pages(cpu_buffer); | ||
3824 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | ||
3825 | 3460 | ||
3826 | atomic_dec(&cpu_buffer->record_disabled); | 3461 | atomic_dec(&cpu_buffer->record_disabled); |
3827 | atomic_dec(&cpu_buffer->buffer->resize_disabled); | ||
3828 | kfree(iter); | 3462 | kfree(iter); |
3829 | } | 3463 | } |
3830 | EXPORT_SYMBOL_GPL(ring_buffer_read_finish); | 3464 | EXPORT_SYMBOL_GPL(ring_buffer_read_finish); |
@@ -3843,7 +3477,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts) | |||
3843 | struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; | 3477 | struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; |
3844 | unsigned long flags; | 3478 | unsigned long flags; |
3845 | 3479 | ||
3846 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | 3480 | spin_lock_irqsave(&cpu_buffer->reader_lock, flags); |
3847 | again: | 3481 | again: |
3848 | event = rb_iter_peek(iter, ts); | 3482 | event = rb_iter_peek(iter, ts); |
3849 | if (!event) | 3483 | if (!event) |
@@ -3854,7 +3488,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts) | |||
3854 | 3488 | ||
3855 | rb_advance_iter(iter); | 3489 | rb_advance_iter(iter); |
3856 | out: | 3490 | out: |
3857 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | 3491 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); |
3858 | 3492 | ||
3859 | return event; | 3493 | return event; |
3860 | } | 3494 | } |
@@ -3864,18 +3498,9 @@ EXPORT_SYMBOL_GPL(ring_buffer_read); | |||
3864 | * ring_buffer_size - return the size of the ring buffer (in bytes) | 3498 | * ring_buffer_size - return the size of the ring buffer (in bytes) |
3865 | * @buffer: The ring buffer. | 3499 | * @buffer: The ring buffer. |
3866 | */ | 3500 | */ |
3867 | unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu) | 3501 | unsigned long ring_buffer_size(struct ring_buffer *buffer) |
3868 | { | 3502 | { |
3869 | /* | 3503 | return BUF_PAGE_SIZE * buffer->pages; |
3870 | * Earlier, this method returned | ||
3871 | * BUF_PAGE_SIZE * buffer->nr_pages | ||
3872 | * Since the nr_pages field is now removed, we have converted this to | ||
3873 | * return the per cpu buffer value. | ||
3874 | */ | ||
3875 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | ||
3876 | return 0; | ||
3877 | |||
3878 | return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages; | ||
3879 | } | 3504 | } |
3880 | EXPORT_SYMBOL_GPL(ring_buffer_size); | 3505 | EXPORT_SYMBOL_GPL(ring_buffer_size); |
3881 | 3506 | ||
@@ -3896,21 +3521,17 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) | |||
3896 | cpu_buffer->commit_page = cpu_buffer->head_page; | 3521 | cpu_buffer->commit_page = cpu_buffer->head_page; |
3897 | 3522 | ||
3898 | INIT_LIST_HEAD(&cpu_buffer->reader_page->list); | 3523 | INIT_LIST_HEAD(&cpu_buffer->reader_page->list); |
3899 | INIT_LIST_HEAD(&cpu_buffer->new_pages); | ||
3900 | local_set(&cpu_buffer->reader_page->write, 0); | 3524 | local_set(&cpu_buffer->reader_page->write, 0); |
3901 | local_set(&cpu_buffer->reader_page->entries, 0); | 3525 | local_set(&cpu_buffer->reader_page->entries, 0); |
3902 | local_set(&cpu_buffer->reader_page->page->commit, 0); | 3526 | local_set(&cpu_buffer->reader_page->page->commit, 0); |
3903 | cpu_buffer->reader_page->read = 0; | 3527 | cpu_buffer->reader_page->read = 0; |
3904 | 3528 | ||
3905 | local_set(&cpu_buffer->entries_bytes, 0); | ||
3906 | local_set(&cpu_buffer->overrun, 0); | ||
3907 | local_set(&cpu_buffer->commit_overrun, 0); | 3529 | local_set(&cpu_buffer->commit_overrun, 0); |
3908 | local_set(&cpu_buffer->dropped_events, 0); | 3530 | local_set(&cpu_buffer->overrun, 0); |
3909 | local_set(&cpu_buffer->entries, 0); | 3531 | local_set(&cpu_buffer->entries, 0); |
3910 | local_set(&cpu_buffer->committing, 0); | 3532 | local_set(&cpu_buffer->committing, 0); |
3911 | local_set(&cpu_buffer->commits, 0); | 3533 | local_set(&cpu_buffer->commits, 0); |
3912 | cpu_buffer->read = 0; | 3534 | cpu_buffer->read = 0; |
3913 | cpu_buffer->read_bytes = 0; | ||
3914 | 3535 | ||
3915 | cpu_buffer->write_stamp = 0; | 3536 | cpu_buffer->write_stamp = 0; |
3916 | cpu_buffer->read_stamp = 0; | 3537 | cpu_buffer->read_stamp = 0; |
@@ -3934,13 +3555,9 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu) | |||
3934 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | 3555 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) |
3935 | return; | 3556 | return; |
3936 | 3557 | ||
3937 | atomic_inc(&buffer->resize_disabled); | ||
3938 | atomic_inc(&cpu_buffer->record_disabled); | 3558 | atomic_inc(&cpu_buffer->record_disabled); |
3939 | 3559 | ||
3940 | /* Make sure all commits have finished */ | 3560 | spin_lock_irqsave(&cpu_buffer->reader_lock, flags); |
3941 | synchronize_sched(); | ||
3942 | |||
3943 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | ||
3944 | 3561 | ||
3945 | if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing))) | 3562 | if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing))) |
3946 | goto out; | 3563 | goto out; |
@@ -3952,10 +3569,9 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu) | |||
3952 | arch_spin_unlock(&cpu_buffer->lock); | 3569 | arch_spin_unlock(&cpu_buffer->lock); |
3953 | 3570 | ||
3954 | out: | 3571 | out: |
3955 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | 3572 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); |
3956 | 3573 | ||
3957 | atomic_dec(&cpu_buffer->record_disabled); | 3574 | atomic_dec(&cpu_buffer->record_disabled); |
3958 | atomic_dec(&buffer->resize_disabled); | ||
3959 | } | 3575 | } |
3960 | EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu); | 3576 | EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu); |
3961 | 3577 | ||
@@ -3991,10 +3607,10 @@ int ring_buffer_empty(struct ring_buffer *buffer) | |||
3991 | cpu_buffer = buffer->buffers[cpu]; | 3607 | cpu_buffer = buffer->buffers[cpu]; |
3992 | local_irq_save(flags); | 3608 | local_irq_save(flags); |
3993 | if (dolock) | 3609 | if (dolock) |
3994 | raw_spin_lock(&cpu_buffer->reader_lock); | 3610 | spin_lock(&cpu_buffer->reader_lock); |
3995 | ret = rb_per_cpu_empty(cpu_buffer); | 3611 | ret = rb_per_cpu_empty(cpu_buffer); |
3996 | if (dolock) | 3612 | if (dolock) |
3997 | raw_spin_unlock(&cpu_buffer->reader_lock); | 3613 | spin_unlock(&cpu_buffer->reader_lock); |
3998 | local_irq_restore(flags); | 3614 | local_irq_restore(flags); |
3999 | 3615 | ||
4000 | if (!ret) | 3616 | if (!ret) |
@@ -4025,10 +3641,10 @@ int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu) | |||
4025 | cpu_buffer = buffer->buffers[cpu]; | 3641 | cpu_buffer = buffer->buffers[cpu]; |
4026 | local_irq_save(flags); | 3642 | local_irq_save(flags); |
4027 | if (dolock) | 3643 | if (dolock) |
4028 | raw_spin_lock(&cpu_buffer->reader_lock); | 3644 | spin_lock(&cpu_buffer->reader_lock); |
4029 | ret = rb_per_cpu_empty(cpu_buffer); | 3645 | ret = rb_per_cpu_empty(cpu_buffer); |
4030 | if (dolock) | 3646 | if (dolock) |
4031 | raw_spin_unlock(&cpu_buffer->reader_lock); | 3647 | spin_unlock(&cpu_buffer->reader_lock); |
4032 | local_irq_restore(flags); | 3648 | local_irq_restore(flags); |
4033 | 3649 | ||
4034 | return ret; | 3650 | return ret; |
@@ -4057,11 +3673,8 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a, | |||
4057 | !cpumask_test_cpu(cpu, buffer_b->cpumask)) | 3673 | !cpumask_test_cpu(cpu, buffer_b->cpumask)) |
4058 | goto out; | 3674 | goto out; |
4059 | 3675 | ||
4060 | cpu_buffer_a = buffer_a->buffers[cpu]; | ||
4061 | cpu_buffer_b = buffer_b->buffers[cpu]; | ||
4062 | |||
4063 | /* At least make sure the two buffers are somewhat the same */ | 3676 | /* At least make sure the two buffers are somewhat the same */ |
4064 | if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages) | 3677 | if (buffer_a->pages != buffer_b->pages) |
4065 | goto out; | 3678 | goto out; |
4066 | 3679 | ||
4067 | ret = -EAGAIN; | 3680 | ret = -EAGAIN; |
@@ -4075,6 +3688,9 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a, | |||
4075 | if (atomic_read(&buffer_b->record_disabled)) | 3688 | if (atomic_read(&buffer_b->record_disabled)) |
4076 | goto out; | 3689 | goto out; |
4077 | 3690 | ||
3691 | cpu_buffer_a = buffer_a->buffers[cpu]; | ||
3692 | cpu_buffer_b = buffer_b->buffers[cpu]; | ||
3693 | |||
4078 | if (atomic_read(&cpu_buffer_a->record_disabled)) | 3694 | if (atomic_read(&cpu_buffer_a->record_disabled)) |
4079 | goto out; | 3695 | goto out; |
4080 | 3696 | ||
@@ -4225,7 +3841,7 @@ int ring_buffer_read_page(struct ring_buffer *buffer, | |||
4225 | if (!bpage) | 3841 | if (!bpage) |
4226 | goto out; | 3842 | goto out; |
4227 | 3843 | ||
4228 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | 3844 | spin_lock_irqsave(&cpu_buffer->reader_lock, flags); |
4229 | 3845 | ||
4230 | reader = rb_get_reader_page(cpu_buffer); | 3846 | reader = rb_get_reader_page(cpu_buffer); |
4231 | if (!reader) | 3847 | if (!reader) |
@@ -4302,7 +3918,6 @@ int ring_buffer_read_page(struct ring_buffer *buffer, | |||
4302 | } else { | 3918 | } else { |
4303 | /* update the entry counter */ | 3919 | /* update the entry counter */ |
4304 | cpu_buffer->read += rb_page_entries(reader); | 3920 | cpu_buffer->read += rb_page_entries(reader); |
4305 | cpu_buffer->read_bytes += BUF_PAGE_SIZE; | ||
4306 | 3921 | ||
4307 | /* swap the pages */ | 3922 | /* swap the pages */ |
4308 | rb_init_page(bpage); | 3923 | rb_init_page(bpage); |
@@ -4349,13 +3964,75 @@ int ring_buffer_read_page(struct ring_buffer *buffer, | |||
4349 | memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit); | 3964 | memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit); |
4350 | 3965 | ||
4351 | out_unlock: | 3966 | out_unlock: |
4352 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | 3967 | spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); |
4353 | 3968 | ||
4354 | out: | 3969 | out: |
4355 | return ret; | 3970 | return ret; |
4356 | } | 3971 | } |
4357 | EXPORT_SYMBOL_GPL(ring_buffer_read_page); | 3972 | EXPORT_SYMBOL_GPL(ring_buffer_read_page); |
4358 | 3973 | ||
3974 | #ifdef CONFIG_TRACING | ||
3975 | static ssize_t | ||
3976 | rb_simple_read(struct file *filp, char __user *ubuf, | ||
3977 | size_t cnt, loff_t *ppos) | ||
3978 | { | ||
3979 | unsigned long *p = filp->private_data; | ||
3980 | char buf[64]; | ||
3981 | int r; | ||
3982 | |||
3983 | if (test_bit(RB_BUFFERS_DISABLED_BIT, p)) | ||
3984 | r = sprintf(buf, "permanently disabled\n"); | ||
3985 | else | ||
3986 | r = sprintf(buf, "%d\n", test_bit(RB_BUFFERS_ON_BIT, p)); | ||
3987 | |||
3988 | return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); | ||
3989 | } | ||
3990 | |||
3991 | static ssize_t | ||
3992 | rb_simple_write(struct file *filp, const char __user *ubuf, | ||
3993 | size_t cnt, loff_t *ppos) | ||
3994 | { | ||
3995 | unsigned long *p = filp->private_data; | ||
3996 | unsigned long val; | ||
3997 | int ret; | ||
3998 | |||
3999 | ret = kstrtoul_from_user(ubuf, cnt, 10, &val); | ||
4000 | if (ret) | ||
4001 | return ret; | ||
4002 | |||
4003 | if (val) | ||
4004 | set_bit(RB_BUFFERS_ON_BIT, p); | ||
4005 | else | ||
4006 | clear_bit(RB_BUFFERS_ON_BIT, p); | ||
4007 | |||
4008 | (*ppos)++; | ||
4009 | |||
4010 | return cnt; | ||
4011 | } | ||
4012 | |||
4013 | static const struct file_operations rb_simple_fops = { | ||
4014 | .open = tracing_open_generic, | ||
4015 | .read = rb_simple_read, | ||
4016 | .write = rb_simple_write, | ||
4017 | .llseek = default_llseek, | ||
4018 | }; | ||
4019 | |||
4020 | |||
4021 | static __init int rb_init_debugfs(void) | ||
4022 | { | ||
4023 | struct dentry *d_tracer; | ||
4024 | |||
4025 | d_tracer = tracing_init_dentry(); | ||
4026 | |||
4027 | trace_create_file("tracing_on", 0644, d_tracer, | ||
4028 | &ring_buffer_flags, &rb_simple_fops); | ||
4029 | |||
4030 | return 0; | ||
4031 | } | ||
4032 | |||
4033 | fs_initcall(rb_init_debugfs); | ||
4034 | #endif | ||
4035 | |||
4359 | #ifdef CONFIG_HOTPLUG_CPU | 4036 | #ifdef CONFIG_HOTPLUG_CPU |
4360 | static int rb_cpu_notify(struct notifier_block *self, | 4037 | static int rb_cpu_notify(struct notifier_block *self, |
4361 | unsigned long action, void *hcpu) | 4038 | unsigned long action, void *hcpu) |
@@ -4363,8 +4040,6 @@ static int rb_cpu_notify(struct notifier_block *self, | |||
4363 | struct ring_buffer *buffer = | 4040 | struct ring_buffer *buffer = |
4364 | container_of(self, struct ring_buffer, cpu_notify); | 4041 | container_of(self, struct ring_buffer, cpu_notify); |
4365 | long cpu = (long)hcpu; | 4042 | long cpu = (long)hcpu; |
4366 | int cpu_i, nr_pages_same; | ||
4367 | unsigned int nr_pages; | ||
4368 | 4043 | ||
4369 | switch (action) { | 4044 | switch (action) { |
4370 | case CPU_UP_PREPARE: | 4045 | case CPU_UP_PREPARE: |
@@ -4372,23 +4047,8 @@ static int rb_cpu_notify(struct notifier_block *self, | |||
4372 | if (cpumask_test_cpu(cpu, buffer->cpumask)) | 4047 | if (cpumask_test_cpu(cpu, buffer->cpumask)) |
4373 | return NOTIFY_OK; | 4048 | return NOTIFY_OK; |
4374 | 4049 | ||
4375 | nr_pages = 0; | ||
4376 | nr_pages_same = 1; | ||
4377 | /* check if all cpu sizes are same */ | ||
4378 | for_each_buffer_cpu(buffer, cpu_i) { | ||
4379 | /* fill in the size from first enabled cpu */ | ||
4380 | if (nr_pages == 0) | ||
4381 | nr_pages = buffer->buffers[cpu_i]->nr_pages; | ||
4382 | if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { | ||
4383 | nr_pages_same = 0; | ||
4384 | break; | ||
4385 | } | ||
4386 | } | ||
4387 | /* allocate minimum pages, user can later expand it */ | ||
4388 | if (!nr_pages_same) | ||
4389 | nr_pages = 2; | ||
4390 | buffer->buffers[cpu] = | 4050 | buffer->buffers[cpu] = |
4391 | rb_allocate_cpu_buffer(buffer, nr_pages, cpu); | 4051 | rb_allocate_cpu_buffer(buffer, cpu); |
4392 | if (!buffer->buffers[cpu]) { | 4052 | if (!buffer->buffers[cpu]) { |
4393 | WARN(1, "failed to allocate ring buffer on CPU %ld\n", | 4053 | WARN(1, "failed to allocate ring buffer on CPU %ld\n", |
4394 | cpu); | 4054 | cpu); |