aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/trace/ring_buffer.c
diff options
context:
space:
mode:
authorJonathan Herman <hermanjl@cs.unc.edu>2013-01-17 16:15:55 -0500
committerJonathan Herman <hermanjl@cs.unc.edu>2013-01-17 16:15:55 -0500
commit8dea78da5cee153b8af9c07a2745f6c55057fe12 (patch)
treea8f4d49d63b1ecc92f2fddceba0655b2472c5bd9 /kernel/trace/ring_buffer.c
parent406089d01562f1e2bf9f089fd7637009ebaad589 (diff)
Patched in Tegra support.
Diffstat (limited to 'kernel/trace/ring_buffer.c')
-rw-r--r--kernel/trace/ring_buffer.c934
1 files changed, 297 insertions, 637 deletions
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index ce8514feedc..731201bf4ac 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -23,8 +23,6 @@
23#include <asm/local.h> 23#include <asm/local.h>
24#include "trace.h" 24#include "trace.h"
25 25
26static void update_pages_handler(struct work_struct *work);
27
28/* 26/*
29 * The ring buffer header is special. We must manually up keep it. 27 * The ring buffer header is special. We must manually up keep it.
30 */ 28 */
@@ -156,12 +154,35 @@ enum {
156 154
157static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; 155static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON;
158 156
159/* Used for individual buffers (after the counter) */
160#define RB_BUFFER_OFF (1 << 20)
161
162#define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data) 157#define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)
163 158
164/** 159/**
160 * tracing_on - enable all tracing buffers
161 *
162 * This function enables all tracing buffers that may have been
163 * disabled with tracing_off.
164 */
165void tracing_on(void)
166{
167 set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
168}
169EXPORT_SYMBOL_GPL(tracing_on);
170
171/**
172 * tracing_off - turn off all tracing buffers
173 *
174 * This function stops all tracing buffers from recording data.
175 * It does not disable any overhead the tracers themselves may
176 * be causing. This function simply causes all recording to
177 * the ring buffers to fail.
178 */
179void tracing_off(void)
180{
181 clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
182}
183EXPORT_SYMBOL_GPL(tracing_off);
184
185/**
165 * tracing_off_permanent - permanently disable ring buffers 186 * tracing_off_permanent - permanently disable ring buffers
166 * 187 *
167 * This function, once called, will disable all ring buffers 188 * This function, once called, will disable all ring buffers
@@ -172,6 +193,15 @@ void tracing_off_permanent(void)
172 set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags); 193 set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);
173} 194}
174 195
196/**
197 * tracing_is_on - show state of ring buffers enabled
198 */
199int tracing_is_on(void)
200{
201 return ring_buffer_flags == RB_BUFFERS_ON;
202}
203EXPORT_SYMBOL_GPL(tracing_is_on);
204
175#define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array)) 205#define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
176#define RB_ALIGNMENT 4U 206#define RB_ALIGNMENT 4U
177#define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) 207#define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
@@ -448,10 +478,9 @@ struct ring_buffer_per_cpu {
448 int cpu; 478 int cpu;
449 atomic_t record_disabled; 479 atomic_t record_disabled;
450 struct ring_buffer *buffer; 480 struct ring_buffer *buffer;
451 raw_spinlock_t reader_lock; /* serialize readers */ 481 spinlock_t reader_lock; /* serialize readers */
452 arch_spinlock_t lock; 482 arch_spinlock_t lock;
453 struct lock_class_key lock_key; 483 struct lock_class_key lock_key;
454 unsigned int nr_pages;
455 struct list_head *pages; 484 struct list_head *pages;
456 struct buffer_page *head_page; /* read from head */ 485 struct buffer_page *head_page; /* read from head */
457 struct buffer_page *tail_page; /* write to tail */ 486 struct buffer_page *tail_page; /* write to tail */
@@ -459,29 +488,21 @@ struct ring_buffer_per_cpu {
459 struct buffer_page *reader_page; 488 struct buffer_page *reader_page;
460 unsigned long lost_events; 489 unsigned long lost_events;
461 unsigned long last_overrun; 490 unsigned long last_overrun;
462 local_t entries_bytes;
463 local_t entries;
464 local_t overrun;
465 local_t commit_overrun; 491 local_t commit_overrun;
466 local_t dropped_events; 492 local_t overrun;
493 local_t entries;
467 local_t committing; 494 local_t committing;
468 local_t commits; 495 local_t commits;
469 unsigned long read; 496 unsigned long read;
470 unsigned long read_bytes;
471 u64 write_stamp; 497 u64 write_stamp;
472 u64 read_stamp; 498 u64 read_stamp;
473 /* ring buffer pages to update, > 0 to add, < 0 to remove */
474 int nr_pages_to_update;
475 struct list_head new_pages; /* new pages to add */
476 struct work_struct update_pages_work;
477 struct completion update_done;
478}; 499};
479 500
480struct ring_buffer { 501struct ring_buffer {
502 unsigned pages;
481 unsigned flags; 503 unsigned flags;
482 int cpus; 504 int cpus;
483 atomic_t record_disabled; 505 atomic_t record_disabled;
484 atomic_t resize_disabled;
485 cpumask_var_t cpumask; 506 cpumask_var_t cpumask;
486 507
487 struct lock_class_key *reader_lock_key; 508 struct lock_class_key *reader_lock_key;
@@ -946,10 +967,6 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
946 struct list_head *head = cpu_buffer->pages; 967 struct list_head *head = cpu_buffer->pages;
947 struct buffer_page *bpage, *tmp; 968 struct buffer_page *bpage, *tmp;
948 969
949 /* Reset the head page if it exists */
950 if (cpu_buffer->head_page)
951 rb_set_head_page(cpu_buffer);
952
953 rb_head_page_deactivate(cpu_buffer); 970 rb_head_page_deactivate(cpu_buffer);
954 971
955 if (RB_WARN_ON(cpu_buffer, head->next->prev != head)) 972 if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
@@ -976,10 +993,14 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
976 return 0; 993 return 0;
977} 994}
978 995
979static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu) 996static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
997 unsigned nr_pages)
980{ 998{
981 int i;
982 struct buffer_page *bpage, *tmp; 999 struct buffer_page *bpage, *tmp;
1000 LIST_HEAD(pages);
1001 unsigned i;
1002
1003 WARN_ON(!nr_pages);
983 1004
984 for (i = 0; i < nr_pages; i++) { 1005 for (i = 0; i < nr_pages; i++) {
985 struct page *page; 1006 struct page *page;
@@ -990,13 +1011,15 @@ static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu)
990 */ 1011 */
991 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), 1012 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
992 GFP_KERNEL | __GFP_NORETRY, 1013 GFP_KERNEL | __GFP_NORETRY,
993 cpu_to_node(cpu)); 1014 cpu_to_node(cpu_buffer->cpu));
994 if (!bpage) 1015 if (!bpage)
995 goto free_pages; 1016 goto free_pages;
996 1017
997 list_add(&bpage->list, pages); 1018 rb_check_bpage(cpu_buffer, bpage);
998 1019
999 page = alloc_pages_node(cpu_to_node(cpu), 1020 list_add(&bpage->list, &pages);
1021
1022 page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu),
1000 GFP_KERNEL | __GFP_NORETRY, 0); 1023 GFP_KERNEL | __GFP_NORETRY, 0);
1001 if (!page) 1024 if (!page)
1002 goto free_pages; 1025 goto free_pages;
@@ -1004,27 +1027,6 @@ static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu)
1004 rb_init_page(bpage->page); 1027 rb_init_page(bpage->page);
1005 } 1028 }
1006 1029
1007 return 0;
1008
1009free_pages:
1010 list_for_each_entry_safe(bpage, tmp, pages, list) {
1011 list_del_init(&bpage->list);
1012 free_buffer_page(bpage);
1013 }
1014
1015 return -ENOMEM;
1016}
1017
1018static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
1019 unsigned nr_pages)
1020{
1021 LIST_HEAD(pages);
1022
1023 WARN_ON(!nr_pages);
1024
1025 if (__rb_allocate_pages(nr_pages, &pages, cpu_buffer->cpu))
1026 return -ENOMEM;
1027
1028 /* 1030 /*
1029 * The ring buffer page list is a circular list that does not 1031 * The ring buffer page list is a circular list that does not
1030 * start and end with a list head. All page list items point to 1032 * start and end with a list head. All page list items point to
@@ -1033,15 +1035,20 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
1033 cpu_buffer->pages = pages.next; 1035 cpu_buffer->pages = pages.next;
1034 list_del(&pages); 1036 list_del(&pages);
1035 1037
1036 cpu_buffer->nr_pages = nr_pages;
1037
1038 rb_check_pages(cpu_buffer); 1038 rb_check_pages(cpu_buffer);
1039 1039
1040 return 0; 1040 return 0;
1041
1042 free_pages:
1043 list_for_each_entry_safe(bpage, tmp, &pages, list) {
1044 list_del_init(&bpage->list);
1045 free_buffer_page(bpage);
1046 }
1047 return -ENOMEM;
1041} 1048}
1042 1049
1043static struct ring_buffer_per_cpu * 1050static struct ring_buffer_per_cpu *
1044rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu) 1051rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
1045{ 1052{
1046 struct ring_buffer_per_cpu *cpu_buffer; 1053 struct ring_buffer_per_cpu *cpu_buffer;
1047 struct buffer_page *bpage; 1054 struct buffer_page *bpage;
@@ -1055,11 +1062,9 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu)
1055 1062
1056 cpu_buffer->cpu = cpu; 1063 cpu_buffer->cpu = cpu;
1057 cpu_buffer->buffer = buffer; 1064 cpu_buffer->buffer = buffer;
1058 raw_spin_lock_init(&cpu_buffer->reader_lock); 1065 spin_lock_init(&cpu_buffer->reader_lock);
1059 lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); 1066 lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);
1060 cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 1067 cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1061 INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler);
1062 init_completion(&cpu_buffer->update_done);
1063 1068
1064 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), 1069 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
1065 GFP_KERNEL, cpu_to_node(cpu)); 1070 GFP_KERNEL, cpu_to_node(cpu));
@@ -1076,9 +1081,8 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu)
1076 rb_init_page(bpage->page); 1081 rb_init_page(bpage->page);
1077 1082
1078 INIT_LIST_HEAD(&cpu_buffer->reader_page->list); 1083 INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
1079 INIT_LIST_HEAD(&cpu_buffer->new_pages);
1080 1084
1081 ret = rb_allocate_pages(cpu_buffer, nr_pages); 1085 ret = rb_allocate_pages(cpu_buffer, buffer->pages);
1082 if (ret < 0) 1086 if (ret < 0)
1083 goto fail_free_reader; 1087 goto fail_free_reader;
1084 1088
@@ -1139,7 +1143,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
1139{ 1143{
1140 struct ring_buffer *buffer; 1144 struct ring_buffer *buffer;
1141 int bsize; 1145 int bsize;
1142 int cpu, nr_pages; 1146 int cpu;
1143 1147
1144 /* keep it in its own cache line */ 1148 /* keep it in its own cache line */
1145 buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), 1149 buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
@@ -1150,14 +1154,14 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
1150 if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) 1154 if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
1151 goto fail_free_buffer; 1155 goto fail_free_buffer;
1152 1156
1153 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); 1157 buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1154 buffer->flags = flags; 1158 buffer->flags = flags;
1155 buffer->clock = trace_clock_local; 1159 buffer->clock = trace_clock_local;
1156 buffer->reader_lock_key = key; 1160 buffer->reader_lock_key = key;
1157 1161
1158 /* need at least two pages */ 1162 /* need at least two pages */
1159 if (nr_pages < 2) 1163 if (buffer->pages < 2)
1160 nr_pages = 2; 1164 buffer->pages = 2;
1161 1165
1162 /* 1166 /*
1163 * In case of non-hotplug cpu, if the ring-buffer is allocated 1167 * In case of non-hotplug cpu, if the ring-buffer is allocated
@@ -1180,7 +1184,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
1180 1184
1181 for_each_buffer_cpu(buffer, cpu) { 1185 for_each_buffer_cpu(buffer, cpu) {
1182 buffer->buffers[cpu] = 1186 buffer->buffers[cpu] =
1183 rb_allocate_cpu_buffer(buffer, nr_pages, cpu); 1187 rb_allocate_cpu_buffer(buffer, cpu);
1184 if (!buffer->buffers[cpu]) 1188 if (!buffer->buffers[cpu])
1185 goto fail_free_buffers; 1189 goto fail_free_buffers;
1186 } 1190 }
@@ -1248,223 +1252,58 @@ void ring_buffer_set_clock(struct ring_buffer *buffer,
1248 1252
1249static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); 1253static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
1250 1254
1251static inline unsigned long rb_page_entries(struct buffer_page *bpage) 1255static void
1252{ 1256rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
1253 return local_read(&bpage->entries) & RB_WRITE_MASK;
1254}
1255
1256static inline unsigned long rb_page_write(struct buffer_page *bpage)
1257{
1258 return local_read(&bpage->write) & RB_WRITE_MASK;
1259}
1260
1261static int
1262rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned int nr_pages)
1263{ 1257{
1264 struct list_head *tail_page, *to_remove, *next_page; 1258 struct buffer_page *bpage;
1265 struct buffer_page *to_remove_page, *tmp_iter_page; 1259 struct list_head *p;
1266 struct buffer_page *last_page, *first_page; 1260 unsigned i;
1267 unsigned int nr_removed;
1268 unsigned long head_bit;
1269 int page_entries;
1270
1271 head_bit = 0;
1272
1273 raw_spin_lock_irq(&cpu_buffer->reader_lock);
1274 atomic_inc(&cpu_buffer->record_disabled);
1275 /*
1276 * We don't race with the readers since we have acquired the reader
1277 * lock. We also don't race with writers after disabling recording.
1278 * This makes it easy to figure out the first and the last page to be
1279 * removed from the list. We unlink all the pages in between including
1280 * the first and last pages. This is done in a busy loop so that we
1281 * lose the least number of traces.
1282 * The pages are freed after we restart recording and unlock readers.
1283 */
1284 tail_page = &cpu_buffer->tail_page->list;
1285
1286 /*
1287 * tail page might be on reader page, we remove the next page
1288 * from the ring buffer
1289 */
1290 if (cpu_buffer->tail_page == cpu_buffer->reader_page)
1291 tail_page = rb_list_head(tail_page->next);
1292 to_remove = tail_page;
1293 1261
1294 /* start of pages to remove */ 1262 spin_lock_irq(&cpu_buffer->reader_lock);
1295 first_page = list_entry(rb_list_head(to_remove->next), 1263 rb_head_page_deactivate(cpu_buffer);
1296 struct buffer_page, list);
1297 1264
1298 for (nr_removed = 0; nr_removed < nr_pages; nr_removed++) { 1265 for (i = 0; i < nr_pages; i++) {
1299 to_remove = rb_list_head(to_remove)->next; 1266 if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)))
1300 head_bit |= (unsigned long)to_remove & RB_PAGE_HEAD; 1267 goto out;
1268 p = cpu_buffer->pages->next;
1269 bpage = list_entry(p, struct buffer_page, list);
1270 list_del_init(&bpage->list);
1271 free_buffer_page(bpage);
1301 } 1272 }
1273 if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)))
1274 goto out;
1302 1275
1303 next_page = rb_list_head(to_remove)->next; 1276 rb_reset_cpu(cpu_buffer);
1304 1277 rb_check_pages(cpu_buffer);
1305 /*
1306 * Now we remove all pages between tail_page and next_page.
1307 * Make sure that we have head_bit value preserved for the
1308 * next page
1309 */
1310 tail_page->next = (struct list_head *)((unsigned long)next_page |
1311 head_bit);
1312 next_page = rb_list_head(next_page);
1313 next_page->prev = tail_page;
1314
1315 /* make sure pages points to a valid page in the ring buffer */
1316 cpu_buffer->pages = next_page;
1317
1318 /* update head page */
1319 if (head_bit)
1320 cpu_buffer->head_page = list_entry(next_page,
1321 struct buffer_page, list);
1322
1323 /*
1324 * change read pointer to make sure any read iterators reset
1325 * themselves
1326 */
1327 cpu_buffer->read = 0;
1328
1329 /* pages are removed, resume tracing and then free the pages */
1330 atomic_dec(&cpu_buffer->record_disabled);
1331 raw_spin_unlock_irq(&cpu_buffer->reader_lock);
1332
1333 RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages));
1334
1335 /* last buffer page to remove */
1336 last_page = list_entry(rb_list_head(to_remove), struct buffer_page,
1337 list);
1338 tmp_iter_page = first_page;
1339
1340 do {
1341 to_remove_page = tmp_iter_page;
1342 rb_inc_page(cpu_buffer, &tmp_iter_page);
1343
1344 /* update the counters */
1345 page_entries = rb_page_entries(to_remove_page);
1346 if (page_entries) {
1347 /*
1348 * If something was added to this page, it was full
1349 * since it is not the tail page. So we deduct the
1350 * bytes consumed in ring buffer from here.
1351 * Increment overrun to account for the lost events.
1352 */
1353 local_add(page_entries, &cpu_buffer->overrun);
1354 local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
1355 }
1356
1357 /*
1358 * We have already removed references to this list item, just
1359 * free up the buffer_page and its page
1360 */
1361 free_buffer_page(to_remove_page);
1362 nr_removed--;
1363
1364 } while (to_remove_page != last_page);
1365
1366 RB_WARN_ON(cpu_buffer, nr_removed);
1367 1278
1368 return nr_removed == 0; 1279out:
1280 spin_unlock_irq(&cpu_buffer->reader_lock);
1369} 1281}
1370 1282
1371static int 1283static void
1372rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer) 1284rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
1285 struct list_head *pages, unsigned nr_pages)
1373{ 1286{
1374 struct list_head *pages = &cpu_buffer->new_pages; 1287 struct buffer_page *bpage;
1375 int retries, success; 1288 struct list_head *p;
1376 1289 unsigned i;
1377 raw_spin_lock_irq(&cpu_buffer->reader_lock);
1378 /*
1379 * We are holding the reader lock, so the reader page won't be swapped
1380 * in the ring buffer. Now we are racing with the writer trying to
1381 * move head page and the tail page.
1382 * We are going to adapt the reader page update process where:
1383 * 1. We first splice the start and end of list of new pages between
1384 * the head page and its previous page.
1385 * 2. We cmpxchg the prev_page->next to point from head page to the
1386 * start of new pages list.
1387 * 3. Finally, we update the head->prev to the end of new list.
1388 *
1389 * We will try this process 10 times, to make sure that we don't keep
1390 * spinning.
1391 */
1392 retries = 10;
1393 success = 0;
1394 while (retries--) {
1395 struct list_head *head_page, *prev_page, *r;
1396 struct list_head *last_page, *first_page;
1397 struct list_head *head_page_with_bit;
1398
1399 head_page = &rb_set_head_page(cpu_buffer)->list;
1400 if (!head_page)
1401 break;
1402 prev_page = head_page->prev;
1403
1404 first_page = pages->next;
1405 last_page = pages->prev;
1406
1407 head_page_with_bit = (struct list_head *)
1408 ((unsigned long)head_page | RB_PAGE_HEAD);
1409
1410 last_page->next = head_page_with_bit;
1411 first_page->prev = prev_page;
1412
1413 r = cmpxchg(&prev_page->next, head_page_with_bit, first_page);
1414 1290
1415 if (r == head_page_with_bit) { 1291 spin_lock_irq(&cpu_buffer->reader_lock);
1416 /* 1292 rb_head_page_deactivate(cpu_buffer);
1417 * yay, we replaced the page pointer to our new list,
1418 * now, we just have to update to head page's prev
1419 * pointer to point to end of list
1420 */
1421 head_page->prev = last_page;
1422 success = 1;
1423 break;
1424 }
1425 }
1426 1293
1427 if (success) 1294 for (i = 0; i < nr_pages; i++) {
1428 INIT_LIST_HEAD(pages); 1295 if (RB_WARN_ON(cpu_buffer, list_empty(pages)))
1429 /* 1296 goto out;
1430 * If we weren't successful in adding in new pages, warn and stop 1297 p = pages->next;
1431 * tracing 1298 bpage = list_entry(p, struct buffer_page, list);
1432 */ 1299 list_del_init(&bpage->list);
1433 RB_WARN_ON(cpu_buffer, !success); 1300 list_add_tail(&bpage->list, cpu_buffer->pages);
1434 raw_spin_unlock_irq(&cpu_buffer->reader_lock);
1435
1436 /* free pages if they weren't inserted */
1437 if (!success) {
1438 struct buffer_page *bpage, *tmp;
1439 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
1440 list) {
1441 list_del_init(&bpage->list);
1442 free_buffer_page(bpage);
1443 }
1444 } 1301 }
1445 return success; 1302 rb_reset_cpu(cpu_buffer);
1446} 1303 rb_check_pages(cpu_buffer);
1447
1448static void rb_update_pages(struct ring_buffer_per_cpu *cpu_buffer)
1449{
1450 int success;
1451
1452 if (cpu_buffer->nr_pages_to_update > 0)
1453 success = rb_insert_pages(cpu_buffer);
1454 else
1455 success = rb_remove_pages(cpu_buffer,
1456 -cpu_buffer->nr_pages_to_update);
1457
1458 if (success)
1459 cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update;
1460}
1461 1304
1462static void update_pages_handler(struct work_struct *work) 1305out:
1463{ 1306 spin_unlock_irq(&cpu_buffer->reader_lock);
1464 struct ring_buffer_per_cpu *cpu_buffer = container_of(work,
1465 struct ring_buffer_per_cpu, update_pages_work);
1466 rb_update_pages(cpu_buffer);
1467 complete(&cpu_buffer->update_done);
1468} 1307}
1469 1308
1470/** 1309/**
@@ -1474,14 +1313,16 @@ static void update_pages_handler(struct work_struct *work)
1474 * 1313 *
1475 * Minimum size is 2 * BUF_PAGE_SIZE. 1314 * Minimum size is 2 * BUF_PAGE_SIZE.
1476 * 1315 *
1477 * Returns 0 on success and < 0 on failure. 1316 * Returns -1 on failure.
1478 */ 1317 */
1479int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size, 1318int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
1480 int cpu_id)
1481{ 1319{
1482 struct ring_buffer_per_cpu *cpu_buffer; 1320 struct ring_buffer_per_cpu *cpu_buffer;
1483 unsigned nr_pages; 1321 unsigned nr_pages, rm_pages, new_pages;
1484 int cpu, err = 0; 1322 struct buffer_page *bpage, *tmp;
1323 unsigned long buffer_size;
1324 LIST_HEAD(pages);
1325 int i, cpu;
1485 1326
1486 /* 1327 /*
1487 * Always succeed at resizing a non-existent buffer: 1328 * Always succeed at resizing a non-existent buffer:
@@ -1489,165 +1330,115 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,
1489 if (!buffer) 1330 if (!buffer)
1490 return size; 1331 return size;
1491 1332
1492 /* Make sure the requested buffer exists */
1493 if (cpu_id != RING_BUFFER_ALL_CPUS &&
1494 !cpumask_test_cpu(cpu_id, buffer->cpumask))
1495 return size;
1496
1497 size = DIV_ROUND_UP(size, BUF_PAGE_SIZE); 1333 size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1498 size *= BUF_PAGE_SIZE; 1334 size *= BUF_PAGE_SIZE;
1335 buffer_size = buffer->pages * BUF_PAGE_SIZE;
1499 1336
1500 /* we need a minimum of two pages */ 1337 /* we need a minimum of two pages */
1501 if (size < BUF_PAGE_SIZE * 2) 1338 if (size < BUF_PAGE_SIZE * 2)
1502 size = BUF_PAGE_SIZE * 2; 1339 size = BUF_PAGE_SIZE * 2;
1503 1340
1504 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); 1341 if (size == buffer_size)
1342 return size;
1505 1343
1506 /* 1344 atomic_inc(&buffer->record_disabled);
1507 * Don't succeed if resizing is disabled, as a reader might be 1345
1508 * manipulating the ring buffer and is expecting a sane state while 1346 /* Make sure all writers are done with this buffer. */
1509 * this is true. 1347 synchronize_sched();
1510 */
1511 if (atomic_read(&buffer->resize_disabled))
1512 return -EBUSY;
1513 1348
1514 /* prevent another thread from changing buffer sizes */
1515 mutex_lock(&buffer->mutex); 1349 mutex_lock(&buffer->mutex);
1350 get_online_cpus();
1516 1351
1517 if (cpu_id == RING_BUFFER_ALL_CPUS) { 1352 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1518 /* calculate the pages to update */
1519 for_each_buffer_cpu(buffer, cpu) {
1520 cpu_buffer = buffer->buffers[cpu];
1521 1353
1522 cpu_buffer->nr_pages_to_update = nr_pages - 1354 if (size < buffer_size) {
1523 cpu_buffer->nr_pages;
1524 /*
1525 * nothing more to do for removing pages or no update
1526 */
1527 if (cpu_buffer->nr_pages_to_update <= 0)
1528 continue;
1529 /*
1530 * to add pages, make sure all new pages can be
1531 * allocated without receiving ENOMEM
1532 */
1533 INIT_LIST_HEAD(&cpu_buffer->new_pages);
1534 if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update,
1535 &cpu_buffer->new_pages, cpu)) {
1536 /* not enough memory for new pages */
1537 err = -ENOMEM;
1538 goto out_err;
1539 }
1540 }
1541 1355
1542 get_online_cpus(); 1356 /* easy case, just free pages */
1543 /* 1357 if (RB_WARN_ON(buffer, nr_pages >= buffer->pages))
1544 * Fire off all the required work handlers 1358 goto out_fail;
1545 * We can't schedule on offline CPUs, but it's not necessary 1359
1546 * since we can change their buffer sizes without any race. 1360 rm_pages = buffer->pages - nr_pages;
1547 */
1548 for_each_buffer_cpu(buffer, cpu) {
1549 cpu_buffer = buffer->buffers[cpu];
1550 if (!cpu_buffer->nr_pages_to_update)
1551 continue;
1552
1553 if (cpu_online(cpu))
1554 schedule_work_on(cpu,
1555 &cpu_buffer->update_pages_work);
1556 else
1557 rb_update_pages(cpu_buffer);
1558 }
1559 1361
1560 /* wait for all the updates to complete */
1561 for_each_buffer_cpu(buffer, cpu) { 1362 for_each_buffer_cpu(buffer, cpu) {
1562 cpu_buffer = buffer->buffers[cpu]; 1363 cpu_buffer = buffer->buffers[cpu];
1563 if (!cpu_buffer->nr_pages_to_update) 1364 rb_remove_pages(cpu_buffer, rm_pages);
1564 continue;
1565
1566 if (cpu_online(cpu))
1567 wait_for_completion(&cpu_buffer->update_done);
1568 cpu_buffer->nr_pages_to_update = 0;
1569 } 1365 }
1366 goto out;
1367 }
1570 1368
1571 put_online_cpus(); 1369 /*
1572 } else { 1370 * This is a bit more difficult. We only want to add pages
1573 /* Make sure this CPU has been intitialized */ 1371 * when we can allocate enough for all CPUs. We do this
1574 if (!cpumask_test_cpu(cpu_id, buffer->cpumask)) 1372 * by allocating all the pages and storing them on a local
1575 goto out; 1373 * link list. If we succeed in our allocation, then we
1576 1374 * add these pages to the cpu_buffers. Otherwise we just free
1577 cpu_buffer = buffer->buffers[cpu_id]; 1375 * them all and return -ENOMEM;
1578 1376 */
1579 if (nr_pages == cpu_buffer->nr_pages) 1377 if (RB_WARN_ON(buffer, nr_pages <= buffer->pages))
1580 goto out; 1378 goto out_fail;
1581 1379
1582 cpu_buffer->nr_pages_to_update = nr_pages - 1380 new_pages = nr_pages - buffer->pages;
1583 cpu_buffer->nr_pages;
1584 1381
1585 INIT_LIST_HEAD(&cpu_buffer->new_pages); 1382 for_each_buffer_cpu(buffer, cpu) {
1586 if (cpu_buffer->nr_pages_to_update > 0 && 1383 for (i = 0; i < new_pages; i++) {
1587 __rb_allocate_pages(cpu_buffer->nr_pages_to_update, 1384 struct page *page;
1588 &cpu_buffer->new_pages, cpu_id)) { 1385 /*
1589 err = -ENOMEM; 1386 * __GFP_NORETRY flag makes sure that the allocation
1590 goto out_err; 1387 * fails gracefully without invoking oom-killer and
1388 * the system is not destabilized.
1389 */
1390 bpage = kzalloc_node(ALIGN(sizeof(*bpage),
1391 cache_line_size()),
1392 GFP_KERNEL | __GFP_NORETRY,
1393 cpu_to_node(cpu));
1394 if (!bpage)
1395 goto free_pages;
1396 list_add(&bpage->list, &pages);
1397 page = alloc_pages_node(cpu_to_node(cpu),
1398 GFP_KERNEL | __GFP_NORETRY, 0);
1399 if (!page)
1400 goto free_pages;
1401 bpage->page = page_address(page);
1402 rb_init_page(bpage->page);
1591 } 1403 }
1592
1593 get_online_cpus();
1594
1595 if (cpu_online(cpu_id)) {
1596 schedule_work_on(cpu_id,
1597 &cpu_buffer->update_pages_work);
1598 wait_for_completion(&cpu_buffer->update_done);
1599 } else
1600 rb_update_pages(cpu_buffer);
1601
1602 cpu_buffer->nr_pages_to_update = 0;
1603 put_online_cpus();
1604 } 1404 }
1605 1405
1606 out: 1406 for_each_buffer_cpu(buffer, cpu) {
1607 /* 1407 cpu_buffer = buffer->buffers[cpu];
1608 * The ring buffer resize can happen with the ring buffer 1408 rb_insert_pages(cpu_buffer, &pages, new_pages);
1609 * enabled, so that the update disturbs the tracing as little
1610 * as possible. But if the buffer is disabled, we do not need
1611 * to worry about that, and we can take the time to verify
1612 * that the buffer is not corrupt.
1613 */
1614 if (atomic_read(&buffer->record_disabled)) {
1615 atomic_inc(&buffer->record_disabled);
1616 /*
1617 * Even though the buffer was disabled, we must make sure
1618 * that it is truly disabled before calling rb_check_pages.
1619 * There could have been a race between checking
1620 * record_disable and incrementing it.
1621 */
1622 synchronize_sched();
1623 for_each_buffer_cpu(buffer, cpu) {
1624 cpu_buffer = buffer->buffers[cpu];
1625 rb_check_pages(cpu_buffer);
1626 }
1627 atomic_dec(&buffer->record_disabled);
1628 } 1409 }
1629 1410
1630 mutex_unlock(&buffer->mutex); 1411 if (RB_WARN_ON(buffer, !list_empty(&pages)))
1631 return size; 1412 goto out_fail;
1632 1413
1633 out_err: 1414 out:
1634 for_each_buffer_cpu(buffer, cpu) { 1415 buffer->pages = nr_pages;
1635 struct buffer_page *bpage, *tmp; 1416 put_online_cpus();
1417 mutex_unlock(&buffer->mutex);
1636 1418
1637 cpu_buffer = buffer->buffers[cpu]; 1419 atomic_dec(&buffer->record_disabled);
1638 cpu_buffer->nr_pages_to_update = 0;
1639 1420
1640 if (list_empty(&cpu_buffer->new_pages)) 1421 return size;
1641 continue;
1642 1422
1643 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, 1423 free_pages:
1644 list) { 1424 list_for_each_entry_safe(bpage, tmp, &pages, list) {
1645 list_del_init(&bpage->list); 1425 list_del_init(&bpage->list);
1646 free_buffer_page(bpage); 1426 free_buffer_page(bpage);
1647 }
1648 } 1427 }
1428 put_online_cpus();
1649 mutex_unlock(&buffer->mutex); 1429 mutex_unlock(&buffer->mutex);
1650 return err; 1430 atomic_dec(&buffer->record_disabled);
1431 return -ENOMEM;
1432
1433 /*
1434 * Something went totally wrong, and we are too paranoid
1435 * to even clean up the mess.
1436 */
1437 out_fail:
1438 put_online_cpus();
1439 mutex_unlock(&buffer->mutex);
1440 atomic_dec(&buffer->record_disabled);
1441 return -1;
1651} 1442}
1652EXPORT_SYMBOL_GPL(ring_buffer_resize); 1443EXPORT_SYMBOL_GPL(ring_buffer_resize);
1653 1444
@@ -1686,11 +1477,21 @@ rb_iter_head_event(struct ring_buffer_iter *iter)
1686 return __rb_page_index(iter->head_page, iter->head); 1477 return __rb_page_index(iter->head_page, iter->head);
1687} 1478}
1688 1479
1480static inline unsigned long rb_page_write(struct buffer_page *bpage)
1481{
1482 return local_read(&bpage->write) & RB_WRITE_MASK;
1483}
1484
1689static inline unsigned rb_page_commit(struct buffer_page *bpage) 1485static inline unsigned rb_page_commit(struct buffer_page *bpage)
1690{ 1486{
1691 return local_read(&bpage->page->commit); 1487 return local_read(&bpage->page->commit);
1692} 1488}
1693 1489
1490static inline unsigned long rb_page_entries(struct buffer_page *bpage)
1491{
1492 return local_read(&bpage->entries) & RB_WRITE_MASK;
1493}
1494
1694/* Size is determined by what has been committed */ 1495/* Size is determined by what has been committed */
1695static inline unsigned rb_page_size(struct buffer_page *bpage) 1496static inline unsigned rb_page_size(struct buffer_page *bpage)
1696{ 1497{
@@ -1739,7 +1540,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
1739 * assign the commit to the tail. 1540 * assign the commit to the tail.
1740 */ 1541 */
1741 again: 1542 again:
1742 max_count = cpu_buffer->nr_pages * 100; 1543 max_count = cpu_buffer->buffer->pages * 100;
1743 1544
1744 while (cpu_buffer->commit_page != cpu_buffer->tail_page) { 1545 while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
1745 if (RB_WARN_ON(cpu_buffer, !(--max_count))) 1546 if (RB_WARN_ON(cpu_buffer, !(--max_count)))
@@ -1823,7 +1624,7 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta)
1823} 1624}
1824 1625
1825/** 1626/**
1826 * rb_update_event - update event type and data 1627 * ring_buffer_update_event - update event type and data
1827 * @event: the even to update 1628 * @event: the even to update
1828 * @type: the type of event 1629 * @type: the type of event
1829 * @length: the size of the event field in the ring buffer 1630 * @length: the size of the event field in the ring buffer
@@ -1907,7 +1708,6 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
1907 * the counters. 1708 * the counters.
1908 */ 1709 */
1909 local_add(entries, &cpu_buffer->overrun); 1710 local_add(entries, &cpu_buffer->overrun);
1910 local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
1911 1711
1912 /* 1712 /*
1913 * The entries will be zeroed out when we move the 1713 * The entries will be zeroed out when we move the
@@ -2063,9 +1863,6 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
2063 event = __rb_page_index(tail_page, tail); 1863 event = __rb_page_index(tail_page, tail);
2064 kmemcheck_annotate_bitfield(event, bitfield); 1864 kmemcheck_annotate_bitfield(event, bitfield);
2065 1865
2066 /* account for padding bytes */
2067 local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes);
2068
2069 /* 1866 /*
2070 * Save the original length to the meta data. 1867 * Save the original length to the meta data.
2071 * This will be used by the reader to add lost event 1868 * This will be used by the reader to add lost event
@@ -2158,10 +1955,8 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
2158 * If we are not in overwrite mode, 1955 * If we are not in overwrite mode,
2159 * this is easy, just stop here. 1956 * this is easy, just stop here.
2160 */ 1957 */
2161 if (!(buffer->flags & RB_FL_OVERWRITE)) { 1958 if (!(buffer->flags & RB_FL_OVERWRITE))
2162 local_inc(&cpu_buffer->dropped_events);
2163 goto out_reset; 1959 goto out_reset;
2164 }
2165 1960
2166 ret = rb_handle_head_page(cpu_buffer, 1961 ret = rb_handle_head_page(cpu_buffer,
2167 tail_page, 1962 tail_page,
@@ -2259,9 +2054,6 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
2259 if (!tail) 2054 if (!tail)
2260 tail_page->page->time_stamp = ts; 2055 tail_page->page->time_stamp = ts;
2261 2056
2262 /* account for these added bytes */
2263 local_add(length, &cpu_buffer->entries_bytes);
2264
2265 return event; 2057 return event;
2266} 2058}
2267 2059
@@ -2284,7 +2076,6 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
2284 if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) { 2076 if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
2285 unsigned long write_mask = 2077 unsigned long write_mask =
2286 local_read(&bpage->write) & ~RB_WRITE_MASK; 2078 local_read(&bpage->write) & ~RB_WRITE_MASK;
2287 unsigned long event_length = rb_event_length(event);
2288 /* 2079 /*
2289 * This is on the tail page. It is possible that 2080 * This is on the tail page. It is possible that
2290 * a write could come in and move the tail page 2081 * a write could come in and move the tail page
@@ -2294,11 +2085,8 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
2294 old_index += write_mask; 2085 old_index += write_mask;
2295 new_index += write_mask; 2086 new_index += write_mask;
2296 index = local_cmpxchg(&bpage->write, old_index, new_index); 2087 index = local_cmpxchg(&bpage->write, old_index, new_index);
2297 if (index == old_index) { 2088 if (index == old_index)
2298 /* update counters */
2299 local_sub(event_length, &cpu_buffer->entries_bytes);
2300 return 1; 2089 return 1;
2301 }
2302 } 2090 }
2303 2091
2304 /* could not discard */ 2092 /* could not discard */
@@ -2725,8 +2513,8 @@ EXPORT_SYMBOL_GPL(ring_buffer_discard_commit);
2725 * and not the length of the event which would hold the header. 2513 * and not the length of the event which would hold the header.
2726 */ 2514 */
2727int ring_buffer_write(struct ring_buffer *buffer, 2515int ring_buffer_write(struct ring_buffer *buffer,
2728 unsigned long length, 2516 unsigned long length,
2729 void *data) 2517 void *data)
2730{ 2518{
2731 struct ring_buffer_per_cpu *cpu_buffer; 2519 struct ring_buffer_per_cpu *cpu_buffer;
2732 struct ring_buffer_event *event; 2520 struct ring_buffer_event *event;
@@ -2818,63 +2606,6 @@ void ring_buffer_record_enable(struct ring_buffer *buffer)
2818EXPORT_SYMBOL_GPL(ring_buffer_record_enable); 2606EXPORT_SYMBOL_GPL(ring_buffer_record_enable);
2819 2607
2820/** 2608/**
2821 * ring_buffer_record_off - stop all writes into the buffer
2822 * @buffer: The ring buffer to stop writes to.
2823 *
2824 * This prevents all writes to the buffer. Any attempt to write
2825 * to the buffer after this will fail and return NULL.
2826 *
2827 * This is different than ring_buffer_record_disable() as
2828 * it works like an on/off switch, where as the disable() version
2829 * must be paired with a enable().
2830 */
2831void ring_buffer_record_off(struct ring_buffer *buffer)
2832{
2833 unsigned int rd;
2834 unsigned int new_rd;
2835
2836 do {
2837 rd = atomic_read(&buffer->record_disabled);
2838 new_rd = rd | RB_BUFFER_OFF;
2839 } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);
2840}
2841EXPORT_SYMBOL_GPL(ring_buffer_record_off);
2842
2843/**
2844 * ring_buffer_record_on - restart writes into the buffer
2845 * @buffer: The ring buffer to start writes to.
2846 *
2847 * This enables all writes to the buffer that was disabled by
2848 * ring_buffer_record_off().
2849 *
2850 * This is different than ring_buffer_record_enable() as
2851 * it works like an on/off switch, where as the enable() version
2852 * must be paired with a disable().
2853 */
2854void ring_buffer_record_on(struct ring_buffer *buffer)
2855{
2856 unsigned int rd;
2857 unsigned int new_rd;
2858
2859 do {
2860 rd = atomic_read(&buffer->record_disabled);
2861 new_rd = rd & ~RB_BUFFER_OFF;
2862 } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);
2863}
2864EXPORT_SYMBOL_GPL(ring_buffer_record_on);
2865
2866/**
2867 * ring_buffer_record_is_on - return true if the ring buffer can write
2868 * @buffer: The ring buffer to see if write is enabled
2869 *
2870 * Returns true if the ring buffer is in a state that it accepts writes.
2871 */
2872int ring_buffer_record_is_on(struct ring_buffer *buffer)
2873{
2874 return !atomic_read(&buffer->record_disabled);
2875}
2876
2877/**
2878 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer 2609 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
2879 * @buffer: The ring buffer to stop writes to. 2610 * @buffer: The ring buffer to stop writes to.
2880 * @cpu: The CPU buffer to stop 2611 * @cpu: The CPU buffer to stop
@@ -2930,59 +2661,6 @@ rb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer)
2930} 2661}
2931 2662
2932/** 2663/**
2933 * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
2934 * @buffer: The ring buffer
2935 * @cpu: The per CPU buffer to read from.
2936 */
2937u64 ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu)
2938{
2939 unsigned long flags;
2940 struct ring_buffer_per_cpu *cpu_buffer;
2941 struct buffer_page *bpage;
2942 u64 ret = 0;
2943
2944 if (!cpumask_test_cpu(cpu, buffer->cpumask))
2945 return 0;
2946
2947 cpu_buffer = buffer->buffers[cpu];
2948 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2949 /*
2950 * if the tail is on reader_page, oldest time stamp is on the reader
2951 * page
2952 */
2953 if (cpu_buffer->tail_page == cpu_buffer->reader_page)
2954 bpage = cpu_buffer->reader_page;
2955 else
2956 bpage = rb_set_head_page(cpu_buffer);
2957 if (bpage)
2958 ret = bpage->page->time_stamp;
2959 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2960
2961 return ret;
2962}
2963EXPORT_SYMBOL_GPL(ring_buffer_oldest_event_ts);
2964
2965/**
2966 * ring_buffer_bytes_cpu - get the number of bytes consumed in a cpu buffer
2967 * @buffer: The ring buffer
2968 * @cpu: The per CPU buffer to read from.
2969 */
2970unsigned long ring_buffer_bytes_cpu(struct ring_buffer *buffer, int cpu)
2971{
2972 struct ring_buffer_per_cpu *cpu_buffer;
2973 unsigned long ret;
2974
2975 if (!cpumask_test_cpu(cpu, buffer->cpumask))
2976 return 0;
2977
2978 cpu_buffer = buffer->buffers[cpu];
2979 ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes;
2980
2981 return ret;
2982}
2983EXPORT_SYMBOL_GPL(ring_buffer_bytes_cpu);
2984
2985/**
2986 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer 2664 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
2987 * @buffer: The ring buffer 2665 * @buffer: The ring buffer
2988 * @cpu: The per CPU buffer to get the entries from. 2666 * @cpu: The per CPU buffer to get the entries from.
@@ -3001,8 +2679,7 @@ unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
3001EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu); 2679EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
3002 2680
3003/** 2681/**
3004 * ring_buffer_overrun_cpu - get the number of overruns caused by the ring 2682 * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer
3005 * buffer wrapping around (only if RB_FL_OVERWRITE is on).
3006 * @buffer: The ring buffer 2683 * @buffer: The ring buffer
3007 * @cpu: The per CPU buffer to get the number of overruns from 2684 * @cpu: The per CPU buffer to get the number of overruns from
3008 */ 2685 */
@@ -3022,9 +2699,7 @@ unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
3022EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu); 2699EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
3023 2700
3024/** 2701/**
3025 * ring_buffer_commit_overrun_cpu - get the number of overruns caused by 2702 * ring_buffer_commit_overrun_cpu - get the number of overruns caused by commits
3026 * commits failing due to the buffer wrapping around while there are uncommitted
3027 * events, such as during an interrupt storm.
3028 * @buffer: The ring buffer 2703 * @buffer: The ring buffer
3029 * @cpu: The per CPU buffer to get the number of overruns from 2704 * @cpu: The per CPU buffer to get the number of overruns from
3030 */ 2705 */
@@ -3045,28 +2720,6 @@ ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
3045EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu); 2720EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu);
3046 2721
3047/** 2722/**
3048 * ring_buffer_dropped_events_cpu - get the number of dropped events caused by
3049 * the ring buffer filling up (only if RB_FL_OVERWRITE is off).
3050 * @buffer: The ring buffer
3051 * @cpu: The per CPU buffer to get the number of overruns from
3052 */
3053unsigned long
3054ring_buffer_dropped_events_cpu(struct ring_buffer *buffer, int cpu)
3055{
3056 struct ring_buffer_per_cpu *cpu_buffer;
3057 unsigned long ret;
3058
3059 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3060 return 0;
3061
3062 cpu_buffer = buffer->buffers[cpu];
3063 ret = local_read(&cpu_buffer->dropped_events);
3064
3065 return ret;
3066}
3067EXPORT_SYMBOL_GPL(ring_buffer_dropped_events_cpu);
3068
3069/**
3070 * ring_buffer_entries - get the number of entries in a buffer 2723 * ring_buffer_entries - get the number of entries in a buffer
3071 * @buffer: The ring buffer 2724 * @buffer: The ring buffer
3072 * 2725 *
@@ -3151,9 +2804,9 @@ void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
3151 2804
3152 cpu_buffer = iter->cpu_buffer; 2805 cpu_buffer = iter->cpu_buffer;
3153 2806
3154 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 2807 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3155 rb_iter_reset(iter); 2808 rb_iter_reset(iter);
3156 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 2809 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3157} 2810}
3158EXPORT_SYMBOL_GPL(ring_buffer_iter_reset); 2811EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
3159 2812
@@ -3274,10 +2927,6 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
3274 if (cpu_buffer->commit_page == cpu_buffer->reader_page) 2927 if (cpu_buffer->commit_page == cpu_buffer->reader_page)
3275 goto out; 2928 goto out;
3276 2929
3277 /* Don't bother swapping if the ring buffer is empty */
3278 if (rb_num_of_entries(cpu_buffer) == 0)
3279 goto out;
3280
3281 /* 2930 /*
3282 * Reset the reader page to size zero. 2931 * Reset the reader page to size zero.
3283 */ 2932 */
@@ -3291,8 +2940,6 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
3291 * Splice the empty reader page into the list around the head. 2940 * Splice the empty reader page into the list around the head.
3292 */ 2941 */
3293 reader = rb_set_head_page(cpu_buffer); 2942 reader = rb_set_head_page(cpu_buffer);
3294 if (!reader)
3295 goto out;
3296 cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next); 2943 cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);
3297 cpu_buffer->reader_page->list.prev = reader->list.prev; 2944 cpu_buffer->reader_page->list.prev = reader->list.prev;
3298 2945
@@ -3618,12 +3265,12 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts,
3618 again: 3265 again:
3619 local_irq_save(flags); 3266 local_irq_save(flags);
3620 if (dolock) 3267 if (dolock)
3621 raw_spin_lock(&cpu_buffer->reader_lock); 3268 spin_lock(&cpu_buffer->reader_lock);
3622 event = rb_buffer_peek(cpu_buffer, ts, lost_events); 3269 event = rb_buffer_peek(cpu_buffer, ts, lost_events);
3623 if (event && event->type_len == RINGBUF_TYPE_PADDING) 3270 if (event && event->type_len == RINGBUF_TYPE_PADDING)
3624 rb_advance_reader(cpu_buffer); 3271 rb_advance_reader(cpu_buffer);
3625 if (dolock) 3272 if (dolock)
3626 raw_spin_unlock(&cpu_buffer->reader_lock); 3273 spin_unlock(&cpu_buffer->reader_lock);
3627 local_irq_restore(flags); 3274 local_irq_restore(flags);
3628 3275
3629 if (event && event->type_len == RINGBUF_TYPE_PADDING) 3276 if (event && event->type_len == RINGBUF_TYPE_PADDING)
@@ -3648,9 +3295,9 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
3648 unsigned long flags; 3295 unsigned long flags;
3649 3296
3650 again: 3297 again:
3651 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 3298 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3652 event = rb_iter_peek(iter, ts); 3299 event = rb_iter_peek(iter, ts);
3653 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 3300 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3654 3301
3655 if (event && event->type_len == RINGBUF_TYPE_PADDING) 3302 if (event && event->type_len == RINGBUF_TYPE_PADDING)
3656 goto again; 3303 goto again;
@@ -3690,7 +3337,7 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts,
3690 cpu_buffer = buffer->buffers[cpu]; 3337 cpu_buffer = buffer->buffers[cpu];
3691 local_irq_save(flags); 3338 local_irq_save(flags);
3692 if (dolock) 3339 if (dolock)
3693 raw_spin_lock(&cpu_buffer->reader_lock); 3340 spin_lock(&cpu_buffer->reader_lock);
3694 3341
3695 event = rb_buffer_peek(cpu_buffer, ts, lost_events); 3342 event = rb_buffer_peek(cpu_buffer, ts, lost_events);
3696 if (event) { 3343 if (event) {
@@ -3699,7 +3346,7 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts,
3699 } 3346 }
3700 3347
3701 if (dolock) 3348 if (dolock)
3702 raw_spin_unlock(&cpu_buffer->reader_lock); 3349 spin_unlock(&cpu_buffer->reader_lock);
3703 local_irq_restore(flags); 3350 local_irq_restore(flags);
3704 3351
3705 out: 3352 out:
@@ -3749,7 +3396,6 @@ ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu)
3749 3396
3750 iter->cpu_buffer = cpu_buffer; 3397 iter->cpu_buffer = cpu_buffer;
3751 3398
3752 atomic_inc(&buffer->resize_disabled);
3753 atomic_inc(&cpu_buffer->record_disabled); 3399 atomic_inc(&cpu_buffer->record_disabled);
3754 3400
3755 return iter; 3401 return iter;
@@ -3792,11 +3438,11 @@ ring_buffer_read_start(struct ring_buffer_iter *iter)
3792 3438
3793 cpu_buffer = iter->cpu_buffer; 3439 cpu_buffer = iter->cpu_buffer;
3794 3440
3795 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 3441 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3796 arch_spin_lock(&cpu_buffer->lock); 3442 arch_spin_lock(&cpu_buffer->lock);
3797 rb_iter_reset(iter); 3443 rb_iter_reset(iter);
3798 arch_spin_unlock(&cpu_buffer->lock); 3444 arch_spin_unlock(&cpu_buffer->lock);
3799 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 3445 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3800} 3446}
3801EXPORT_SYMBOL_GPL(ring_buffer_read_start); 3447EXPORT_SYMBOL_GPL(ring_buffer_read_start);
3802 3448
@@ -3811,20 +3457,8 @@ void
3811ring_buffer_read_finish(struct ring_buffer_iter *iter) 3457ring_buffer_read_finish(struct ring_buffer_iter *iter)
3812{ 3458{
3813 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 3459 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
3814 unsigned long flags;
3815
3816 /*
3817 * Ring buffer is disabled from recording, here's a good place
3818 * to check the integrity of the ring buffer.
3819 * Must prevent readers from trying to read, as the check
3820 * clears the HEAD page and readers require it.
3821 */
3822 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3823 rb_check_pages(cpu_buffer);
3824 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3825 3460
3826 atomic_dec(&cpu_buffer->record_disabled); 3461 atomic_dec(&cpu_buffer->record_disabled);
3827 atomic_dec(&cpu_buffer->buffer->resize_disabled);
3828 kfree(iter); 3462 kfree(iter);
3829} 3463}
3830EXPORT_SYMBOL_GPL(ring_buffer_read_finish); 3464EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
@@ -3843,7 +3477,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
3843 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 3477 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
3844 unsigned long flags; 3478 unsigned long flags;
3845 3479
3846 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 3480 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3847 again: 3481 again:
3848 event = rb_iter_peek(iter, ts); 3482 event = rb_iter_peek(iter, ts);
3849 if (!event) 3483 if (!event)
@@ -3854,7 +3488,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
3854 3488
3855 rb_advance_iter(iter); 3489 rb_advance_iter(iter);
3856 out: 3490 out:
3857 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 3491 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3858 3492
3859 return event; 3493 return event;
3860} 3494}
@@ -3864,18 +3498,9 @@ EXPORT_SYMBOL_GPL(ring_buffer_read);
3864 * ring_buffer_size - return the size of the ring buffer (in bytes) 3498 * ring_buffer_size - return the size of the ring buffer (in bytes)
3865 * @buffer: The ring buffer. 3499 * @buffer: The ring buffer.
3866 */ 3500 */
3867unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu) 3501unsigned long ring_buffer_size(struct ring_buffer *buffer)
3868{ 3502{
3869 /* 3503 return BUF_PAGE_SIZE * buffer->pages;
3870 * Earlier, this method returned
3871 * BUF_PAGE_SIZE * buffer->nr_pages
3872 * Since the nr_pages field is now removed, we have converted this to
3873 * return the per cpu buffer value.
3874 */
3875 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3876 return 0;
3877
3878 return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;
3879} 3504}
3880EXPORT_SYMBOL_GPL(ring_buffer_size); 3505EXPORT_SYMBOL_GPL(ring_buffer_size);
3881 3506
@@ -3896,21 +3521,17 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
3896 cpu_buffer->commit_page = cpu_buffer->head_page; 3521 cpu_buffer->commit_page = cpu_buffer->head_page;
3897 3522
3898 INIT_LIST_HEAD(&cpu_buffer->reader_page->list); 3523 INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
3899 INIT_LIST_HEAD(&cpu_buffer->new_pages);
3900 local_set(&cpu_buffer->reader_page->write, 0); 3524 local_set(&cpu_buffer->reader_page->write, 0);
3901 local_set(&cpu_buffer->reader_page->entries, 0); 3525 local_set(&cpu_buffer->reader_page->entries, 0);
3902 local_set(&cpu_buffer->reader_page->page->commit, 0); 3526 local_set(&cpu_buffer->reader_page->page->commit, 0);
3903 cpu_buffer->reader_page->read = 0; 3527 cpu_buffer->reader_page->read = 0;
3904 3528
3905 local_set(&cpu_buffer->entries_bytes, 0);
3906 local_set(&cpu_buffer->overrun, 0);
3907 local_set(&cpu_buffer->commit_overrun, 0); 3529 local_set(&cpu_buffer->commit_overrun, 0);
3908 local_set(&cpu_buffer->dropped_events, 0); 3530 local_set(&cpu_buffer->overrun, 0);
3909 local_set(&cpu_buffer->entries, 0); 3531 local_set(&cpu_buffer->entries, 0);
3910 local_set(&cpu_buffer->committing, 0); 3532 local_set(&cpu_buffer->committing, 0);
3911 local_set(&cpu_buffer->commits, 0); 3533 local_set(&cpu_buffer->commits, 0);
3912 cpu_buffer->read = 0; 3534 cpu_buffer->read = 0;
3913 cpu_buffer->read_bytes = 0;
3914 3535
3915 cpu_buffer->write_stamp = 0; 3536 cpu_buffer->write_stamp = 0;
3916 cpu_buffer->read_stamp = 0; 3537 cpu_buffer->read_stamp = 0;
@@ -3934,13 +3555,9 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
3934 if (!cpumask_test_cpu(cpu, buffer->cpumask)) 3555 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3935 return; 3556 return;
3936 3557
3937 atomic_inc(&buffer->resize_disabled);
3938 atomic_inc(&cpu_buffer->record_disabled); 3558 atomic_inc(&cpu_buffer->record_disabled);
3939 3559
3940 /* Make sure all commits have finished */ 3560 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3941 synchronize_sched();
3942
3943 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3944 3561
3945 if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing))) 3562 if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))
3946 goto out; 3563 goto out;
@@ -3952,10 +3569,9 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
3952 arch_spin_unlock(&cpu_buffer->lock); 3569 arch_spin_unlock(&cpu_buffer->lock);
3953 3570
3954 out: 3571 out:
3955 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 3572 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3956 3573
3957 atomic_dec(&cpu_buffer->record_disabled); 3574 atomic_dec(&cpu_buffer->record_disabled);
3958 atomic_dec(&buffer->resize_disabled);
3959} 3575}
3960EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu); 3576EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
3961 3577
@@ -3991,10 +3607,10 @@ int ring_buffer_empty(struct ring_buffer *buffer)
3991 cpu_buffer = buffer->buffers[cpu]; 3607 cpu_buffer = buffer->buffers[cpu];
3992 local_irq_save(flags); 3608 local_irq_save(flags);
3993 if (dolock) 3609 if (dolock)
3994 raw_spin_lock(&cpu_buffer->reader_lock); 3610 spin_lock(&cpu_buffer->reader_lock);
3995 ret = rb_per_cpu_empty(cpu_buffer); 3611 ret = rb_per_cpu_empty(cpu_buffer);
3996 if (dolock) 3612 if (dolock)
3997 raw_spin_unlock(&cpu_buffer->reader_lock); 3613 spin_unlock(&cpu_buffer->reader_lock);
3998 local_irq_restore(flags); 3614 local_irq_restore(flags);
3999 3615
4000 if (!ret) 3616 if (!ret)
@@ -4025,10 +3641,10 @@ int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
4025 cpu_buffer = buffer->buffers[cpu]; 3641 cpu_buffer = buffer->buffers[cpu];
4026 local_irq_save(flags); 3642 local_irq_save(flags);
4027 if (dolock) 3643 if (dolock)
4028 raw_spin_lock(&cpu_buffer->reader_lock); 3644 spin_lock(&cpu_buffer->reader_lock);
4029 ret = rb_per_cpu_empty(cpu_buffer); 3645 ret = rb_per_cpu_empty(cpu_buffer);
4030 if (dolock) 3646 if (dolock)
4031 raw_spin_unlock(&cpu_buffer->reader_lock); 3647 spin_unlock(&cpu_buffer->reader_lock);
4032 local_irq_restore(flags); 3648 local_irq_restore(flags);
4033 3649
4034 return ret; 3650 return ret;
@@ -4057,11 +3673,8 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
4057 !cpumask_test_cpu(cpu, buffer_b->cpumask)) 3673 !cpumask_test_cpu(cpu, buffer_b->cpumask))
4058 goto out; 3674 goto out;
4059 3675
4060 cpu_buffer_a = buffer_a->buffers[cpu];
4061 cpu_buffer_b = buffer_b->buffers[cpu];
4062
4063 /* At least make sure the two buffers are somewhat the same */ 3676 /* At least make sure the two buffers are somewhat the same */
4064 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages) 3677 if (buffer_a->pages != buffer_b->pages)
4065 goto out; 3678 goto out;
4066 3679
4067 ret = -EAGAIN; 3680 ret = -EAGAIN;
@@ -4075,6 +3688,9 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
4075 if (atomic_read(&buffer_b->record_disabled)) 3688 if (atomic_read(&buffer_b->record_disabled))
4076 goto out; 3689 goto out;
4077 3690
3691 cpu_buffer_a = buffer_a->buffers[cpu];
3692 cpu_buffer_b = buffer_b->buffers[cpu];
3693
4078 if (atomic_read(&cpu_buffer_a->record_disabled)) 3694 if (atomic_read(&cpu_buffer_a->record_disabled))
4079 goto out; 3695 goto out;
4080 3696
@@ -4225,7 +3841,7 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
4225 if (!bpage) 3841 if (!bpage)
4226 goto out; 3842 goto out;
4227 3843
4228 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 3844 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4229 3845
4230 reader = rb_get_reader_page(cpu_buffer); 3846 reader = rb_get_reader_page(cpu_buffer);
4231 if (!reader) 3847 if (!reader)
@@ -4302,7 +3918,6 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
4302 } else { 3918 } else {
4303 /* update the entry counter */ 3919 /* update the entry counter */
4304 cpu_buffer->read += rb_page_entries(reader); 3920 cpu_buffer->read += rb_page_entries(reader);
4305 cpu_buffer->read_bytes += BUF_PAGE_SIZE;
4306 3921
4307 /* swap the pages */ 3922 /* swap the pages */
4308 rb_init_page(bpage); 3923 rb_init_page(bpage);
@@ -4349,13 +3964,75 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
4349 memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit); 3964 memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit);
4350 3965
4351 out_unlock: 3966 out_unlock:
4352 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 3967 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4353 3968
4354 out: 3969 out:
4355 return ret; 3970 return ret;
4356} 3971}
4357EXPORT_SYMBOL_GPL(ring_buffer_read_page); 3972EXPORT_SYMBOL_GPL(ring_buffer_read_page);
4358 3973
3974#ifdef CONFIG_TRACING
3975static ssize_t
3976rb_simple_read(struct file *filp, char __user *ubuf,
3977 size_t cnt, loff_t *ppos)
3978{
3979 unsigned long *p = filp->private_data;
3980 char buf[64];
3981 int r;
3982
3983 if (test_bit(RB_BUFFERS_DISABLED_BIT, p))
3984 r = sprintf(buf, "permanently disabled\n");
3985 else
3986 r = sprintf(buf, "%d\n", test_bit(RB_BUFFERS_ON_BIT, p));
3987
3988 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3989}
3990
3991static ssize_t
3992rb_simple_write(struct file *filp, const char __user *ubuf,
3993 size_t cnt, loff_t *ppos)
3994{
3995 unsigned long *p = filp->private_data;
3996 unsigned long val;
3997 int ret;
3998
3999 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4000 if (ret)
4001 return ret;
4002
4003 if (val)
4004 set_bit(RB_BUFFERS_ON_BIT, p);
4005 else
4006 clear_bit(RB_BUFFERS_ON_BIT, p);
4007
4008 (*ppos)++;
4009
4010 return cnt;
4011}
4012
4013static const struct file_operations rb_simple_fops = {
4014 .open = tracing_open_generic,
4015 .read = rb_simple_read,
4016 .write = rb_simple_write,
4017 .llseek = default_llseek,
4018};
4019
4020
4021static __init int rb_init_debugfs(void)
4022{
4023 struct dentry *d_tracer;
4024
4025 d_tracer = tracing_init_dentry();
4026
4027 trace_create_file("tracing_on", 0644, d_tracer,
4028 &ring_buffer_flags, &rb_simple_fops);
4029
4030 return 0;
4031}
4032
4033fs_initcall(rb_init_debugfs);
4034#endif
4035
4359#ifdef CONFIG_HOTPLUG_CPU 4036#ifdef CONFIG_HOTPLUG_CPU
4360static int rb_cpu_notify(struct notifier_block *self, 4037static int rb_cpu_notify(struct notifier_block *self,
4361 unsigned long action, void *hcpu) 4038 unsigned long action, void *hcpu)
@@ -4363,8 +4040,6 @@ static int rb_cpu_notify(struct notifier_block *self,
4363 struct ring_buffer *buffer = 4040 struct ring_buffer *buffer =
4364 container_of(self, struct ring_buffer, cpu_notify); 4041 container_of(self, struct ring_buffer, cpu_notify);
4365 long cpu = (long)hcpu; 4042 long cpu = (long)hcpu;
4366 int cpu_i, nr_pages_same;
4367 unsigned int nr_pages;
4368 4043
4369 switch (action) { 4044 switch (action) {
4370 case CPU_UP_PREPARE: 4045 case CPU_UP_PREPARE:
@@ -4372,23 +4047,8 @@ static int rb_cpu_notify(struct notifier_block *self,
4372 if (cpumask_test_cpu(cpu, buffer->cpumask)) 4047 if (cpumask_test_cpu(cpu, buffer->cpumask))
4373 return NOTIFY_OK; 4048 return NOTIFY_OK;
4374 4049
4375 nr_pages = 0;
4376 nr_pages_same = 1;
4377 /* check if all cpu sizes are same */
4378 for_each_buffer_cpu(buffer, cpu_i) {
4379 /* fill in the size from first enabled cpu */
4380 if (nr_pages == 0)
4381 nr_pages = buffer->buffers[cpu_i]->nr_pages;
4382 if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {
4383 nr_pages_same = 0;
4384 break;
4385 }
4386 }
4387 /* allocate minimum pages, user can later expand it */
4388 if (!nr_pages_same)
4389 nr_pages = 2;
4390 buffer->buffers[cpu] = 4050 buffer->buffers[cpu] =
4391 rb_allocate_cpu_buffer(buffer, nr_pages, cpu); 4051 rb_allocate_cpu_buffer(buffer, cpu);
4392 if (!buffer->buffers[cpu]) { 4052 if (!buffer->buffers[cpu]) {
4393 WARN(1, "failed to allocate ring buffer on CPU %ld\n", 4053 WARN(1, "failed to allocate ring buffer on CPU %ld\n",
4394 cpu); 4054 cpu);