diff options
Diffstat (limited to 'kernel/trace')
-rw-r--r-- | kernel/trace/ftrace.c | 44 | ||||
-rw-r--r-- | kernel/trace/ring_buffer.c | 248 | ||||
-rw-r--r-- | kernel/trace/trace.c | 453 | ||||
-rw-r--r-- | kernel/trace/trace.h | 4 | ||||
-rw-r--r-- | kernel/trace/trace_printk.c | 4 |
5 files changed, 453 insertions, 300 deletions
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 0fa92f677c9..cf81f27ce6c 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
@@ -2469,57 +2469,35 @@ static int | |||
2469 | ftrace_avail_open(struct inode *inode, struct file *file) | 2469 | ftrace_avail_open(struct inode *inode, struct file *file) |
2470 | { | 2470 | { |
2471 | struct ftrace_iterator *iter; | 2471 | struct ftrace_iterator *iter; |
2472 | int ret; | ||
2473 | 2472 | ||
2474 | if (unlikely(ftrace_disabled)) | 2473 | if (unlikely(ftrace_disabled)) |
2475 | return -ENODEV; | 2474 | return -ENODEV; |
2476 | 2475 | ||
2477 | iter = kzalloc(sizeof(*iter), GFP_KERNEL); | 2476 | iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); |
2478 | if (!iter) | 2477 | if (iter) { |
2479 | return -ENOMEM; | 2478 | iter->pg = ftrace_pages_start; |
2480 | 2479 | iter->ops = &global_ops; | |
2481 | iter->pg = ftrace_pages_start; | ||
2482 | iter->ops = &global_ops; | ||
2483 | |||
2484 | ret = seq_open(file, &show_ftrace_seq_ops); | ||
2485 | if (!ret) { | ||
2486 | struct seq_file *m = file->private_data; | ||
2487 | |||
2488 | m->private = iter; | ||
2489 | } else { | ||
2490 | kfree(iter); | ||
2491 | } | 2480 | } |
2492 | 2481 | ||
2493 | return ret; | 2482 | return iter ? 0 : -ENOMEM; |
2494 | } | 2483 | } |
2495 | 2484 | ||
2496 | static int | 2485 | static int |
2497 | ftrace_enabled_open(struct inode *inode, struct file *file) | 2486 | ftrace_enabled_open(struct inode *inode, struct file *file) |
2498 | { | 2487 | { |
2499 | struct ftrace_iterator *iter; | 2488 | struct ftrace_iterator *iter; |
2500 | int ret; | ||
2501 | 2489 | ||
2502 | if (unlikely(ftrace_disabled)) | 2490 | if (unlikely(ftrace_disabled)) |
2503 | return -ENODEV; | 2491 | return -ENODEV; |
2504 | 2492 | ||
2505 | iter = kzalloc(sizeof(*iter), GFP_KERNEL); | 2493 | iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); |
2506 | if (!iter) | 2494 | if (iter) { |
2507 | return -ENOMEM; | 2495 | iter->pg = ftrace_pages_start; |
2508 | 2496 | iter->flags = FTRACE_ITER_ENABLED; | |
2509 | iter->pg = ftrace_pages_start; | 2497 | iter->ops = &global_ops; |
2510 | iter->flags = FTRACE_ITER_ENABLED; | ||
2511 | iter->ops = &global_ops; | ||
2512 | |||
2513 | ret = seq_open(file, &show_ftrace_seq_ops); | ||
2514 | if (!ret) { | ||
2515 | struct seq_file *m = file->private_data; | ||
2516 | |||
2517 | m->private = iter; | ||
2518 | } else { | ||
2519 | kfree(iter); | ||
2520 | } | 2498 | } |
2521 | 2499 | ||
2522 | return ret; | 2500 | return iter ? 0 : -ENOMEM; |
2523 | } | 2501 | } |
2524 | 2502 | ||
2525 | static void ftrace_filter_reset(struct ftrace_hash *hash) | 2503 | static void ftrace_filter_reset(struct ftrace_hash *hash) |
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index cf8d11e91ef..2d5eb332082 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
@@ -449,6 +449,7 @@ struct ring_buffer_per_cpu { | |||
449 | raw_spinlock_t reader_lock; /* serialize readers */ | 449 | raw_spinlock_t reader_lock; /* serialize readers */ |
450 | arch_spinlock_t lock; | 450 | arch_spinlock_t lock; |
451 | struct lock_class_key lock_key; | 451 | struct lock_class_key lock_key; |
452 | unsigned int nr_pages; | ||
452 | struct list_head *pages; | 453 | struct list_head *pages; |
453 | struct buffer_page *head_page; /* read from head */ | 454 | struct buffer_page *head_page; /* read from head */ |
454 | struct buffer_page *tail_page; /* write to tail */ | 455 | struct buffer_page *tail_page; /* write to tail */ |
@@ -466,10 +467,12 @@ struct ring_buffer_per_cpu { | |||
466 | unsigned long read_bytes; | 467 | unsigned long read_bytes; |
467 | u64 write_stamp; | 468 | u64 write_stamp; |
468 | u64 read_stamp; | 469 | u64 read_stamp; |
470 | /* ring buffer pages to update, > 0 to add, < 0 to remove */ | ||
471 | int nr_pages_to_update; | ||
472 | struct list_head new_pages; /* new pages to add */ | ||
469 | }; | 473 | }; |
470 | 474 | ||
471 | struct ring_buffer { | 475 | struct ring_buffer { |
472 | unsigned pages; | ||
473 | unsigned flags; | 476 | unsigned flags; |
474 | int cpus; | 477 | int cpus; |
475 | atomic_t record_disabled; | 478 | atomic_t record_disabled; |
@@ -963,14 +966,10 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) | |||
963 | return 0; | 966 | return 0; |
964 | } | 967 | } |
965 | 968 | ||
966 | static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, | 969 | static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu) |
967 | unsigned nr_pages) | ||
968 | { | 970 | { |
971 | int i; | ||
969 | struct buffer_page *bpage, *tmp; | 972 | struct buffer_page *bpage, *tmp; |
970 | LIST_HEAD(pages); | ||
971 | unsigned i; | ||
972 | |||
973 | WARN_ON(!nr_pages); | ||
974 | 973 | ||
975 | for (i = 0; i < nr_pages; i++) { | 974 | for (i = 0; i < nr_pages; i++) { |
976 | struct page *page; | 975 | struct page *page; |
@@ -981,15 +980,13 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, | |||
981 | */ | 980 | */ |
982 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), | 981 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), |
983 | GFP_KERNEL | __GFP_NORETRY, | 982 | GFP_KERNEL | __GFP_NORETRY, |
984 | cpu_to_node(cpu_buffer->cpu)); | 983 | cpu_to_node(cpu)); |
985 | if (!bpage) | 984 | if (!bpage) |
986 | goto free_pages; | 985 | goto free_pages; |
987 | 986 | ||
988 | rb_check_bpage(cpu_buffer, bpage); | 987 | list_add(&bpage->list, pages); |
989 | 988 | ||
990 | list_add(&bpage->list, &pages); | 989 | page = alloc_pages_node(cpu_to_node(cpu), |
991 | |||
992 | page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), | ||
993 | GFP_KERNEL | __GFP_NORETRY, 0); | 990 | GFP_KERNEL | __GFP_NORETRY, 0); |
994 | if (!page) | 991 | if (!page) |
995 | goto free_pages; | 992 | goto free_pages; |
@@ -997,6 +994,27 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, | |||
997 | rb_init_page(bpage->page); | 994 | rb_init_page(bpage->page); |
998 | } | 995 | } |
999 | 996 | ||
997 | return 0; | ||
998 | |||
999 | free_pages: | ||
1000 | list_for_each_entry_safe(bpage, tmp, pages, list) { | ||
1001 | list_del_init(&bpage->list); | ||
1002 | free_buffer_page(bpage); | ||
1003 | } | ||
1004 | |||
1005 | return -ENOMEM; | ||
1006 | } | ||
1007 | |||
1008 | static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, | ||
1009 | unsigned nr_pages) | ||
1010 | { | ||
1011 | LIST_HEAD(pages); | ||
1012 | |||
1013 | WARN_ON(!nr_pages); | ||
1014 | |||
1015 | if (__rb_allocate_pages(nr_pages, &pages, cpu_buffer->cpu)) | ||
1016 | return -ENOMEM; | ||
1017 | |||
1000 | /* | 1018 | /* |
1001 | * The ring buffer page list is a circular list that does not | 1019 | * The ring buffer page list is a circular list that does not |
1002 | * start and end with a list head. All page list items point to | 1020 | * start and end with a list head. All page list items point to |
@@ -1005,20 +1023,15 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, | |||
1005 | cpu_buffer->pages = pages.next; | 1023 | cpu_buffer->pages = pages.next; |
1006 | list_del(&pages); | 1024 | list_del(&pages); |
1007 | 1025 | ||
1026 | cpu_buffer->nr_pages = nr_pages; | ||
1027 | |||
1008 | rb_check_pages(cpu_buffer); | 1028 | rb_check_pages(cpu_buffer); |
1009 | 1029 | ||
1010 | return 0; | 1030 | return 0; |
1011 | |||
1012 | free_pages: | ||
1013 | list_for_each_entry_safe(bpage, tmp, &pages, list) { | ||
1014 | list_del_init(&bpage->list); | ||
1015 | free_buffer_page(bpage); | ||
1016 | } | ||
1017 | return -ENOMEM; | ||
1018 | } | 1031 | } |
1019 | 1032 | ||
1020 | static struct ring_buffer_per_cpu * | 1033 | static struct ring_buffer_per_cpu * |
1021 | rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) | 1034 | rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu) |
1022 | { | 1035 | { |
1023 | struct ring_buffer_per_cpu *cpu_buffer; | 1036 | struct ring_buffer_per_cpu *cpu_buffer; |
1024 | struct buffer_page *bpage; | 1037 | struct buffer_page *bpage; |
@@ -1052,7 +1065,7 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) | |||
1052 | 1065 | ||
1053 | INIT_LIST_HEAD(&cpu_buffer->reader_page->list); | 1066 | INIT_LIST_HEAD(&cpu_buffer->reader_page->list); |
1054 | 1067 | ||
1055 | ret = rb_allocate_pages(cpu_buffer, buffer->pages); | 1068 | ret = rb_allocate_pages(cpu_buffer, nr_pages); |
1056 | if (ret < 0) | 1069 | if (ret < 0) |
1057 | goto fail_free_reader; | 1070 | goto fail_free_reader; |
1058 | 1071 | ||
@@ -1113,7 +1126,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, | |||
1113 | { | 1126 | { |
1114 | struct ring_buffer *buffer; | 1127 | struct ring_buffer *buffer; |
1115 | int bsize; | 1128 | int bsize; |
1116 | int cpu; | 1129 | int cpu, nr_pages; |
1117 | 1130 | ||
1118 | /* keep it in its own cache line */ | 1131 | /* keep it in its own cache line */ |
1119 | buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), | 1132 | buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), |
@@ -1124,14 +1137,14 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, | |||
1124 | if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) | 1137 | if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) |
1125 | goto fail_free_buffer; | 1138 | goto fail_free_buffer; |
1126 | 1139 | ||
1127 | buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); | 1140 | nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); |
1128 | buffer->flags = flags; | 1141 | buffer->flags = flags; |
1129 | buffer->clock = trace_clock_local; | 1142 | buffer->clock = trace_clock_local; |
1130 | buffer->reader_lock_key = key; | 1143 | buffer->reader_lock_key = key; |
1131 | 1144 | ||
1132 | /* need at least two pages */ | 1145 | /* need at least two pages */ |
1133 | if (buffer->pages < 2) | 1146 | if (nr_pages < 2) |
1134 | buffer->pages = 2; | 1147 | nr_pages = 2; |
1135 | 1148 | ||
1136 | /* | 1149 | /* |
1137 | * In case of non-hotplug cpu, if the ring-buffer is allocated | 1150 | * In case of non-hotplug cpu, if the ring-buffer is allocated |
@@ -1154,7 +1167,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, | |||
1154 | 1167 | ||
1155 | for_each_buffer_cpu(buffer, cpu) { | 1168 | for_each_buffer_cpu(buffer, cpu) { |
1156 | buffer->buffers[cpu] = | 1169 | buffer->buffers[cpu] = |
1157 | rb_allocate_cpu_buffer(buffer, cpu); | 1170 | rb_allocate_cpu_buffer(buffer, nr_pages, cpu); |
1158 | if (!buffer->buffers[cpu]) | 1171 | if (!buffer->buffers[cpu]) |
1159 | goto fail_free_buffers; | 1172 | goto fail_free_buffers; |
1160 | } | 1173 | } |
@@ -1276,6 +1289,18 @@ out: | |||
1276 | raw_spin_unlock_irq(&cpu_buffer->reader_lock); | 1289 | raw_spin_unlock_irq(&cpu_buffer->reader_lock); |
1277 | } | 1290 | } |
1278 | 1291 | ||
1292 | static void update_pages_handler(struct ring_buffer_per_cpu *cpu_buffer) | ||
1293 | { | ||
1294 | if (cpu_buffer->nr_pages_to_update > 0) | ||
1295 | rb_insert_pages(cpu_buffer, &cpu_buffer->new_pages, | ||
1296 | cpu_buffer->nr_pages_to_update); | ||
1297 | else | ||
1298 | rb_remove_pages(cpu_buffer, -cpu_buffer->nr_pages_to_update); | ||
1299 | cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update; | ||
1300 | /* reset this value */ | ||
1301 | cpu_buffer->nr_pages_to_update = 0; | ||
1302 | } | ||
1303 | |||
1279 | /** | 1304 | /** |
1280 | * ring_buffer_resize - resize the ring buffer | 1305 | * ring_buffer_resize - resize the ring buffer |
1281 | * @buffer: the buffer to resize. | 1306 | * @buffer: the buffer to resize. |
@@ -1285,14 +1310,12 @@ out: | |||
1285 | * | 1310 | * |
1286 | * Returns -1 on failure. | 1311 | * Returns -1 on failure. |
1287 | */ | 1312 | */ |
1288 | int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) | 1313 | int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size, |
1314 | int cpu_id) | ||
1289 | { | 1315 | { |
1290 | struct ring_buffer_per_cpu *cpu_buffer; | 1316 | struct ring_buffer_per_cpu *cpu_buffer; |
1291 | unsigned nr_pages, rm_pages, new_pages; | 1317 | unsigned nr_pages; |
1292 | struct buffer_page *bpage, *tmp; | 1318 | int cpu; |
1293 | unsigned long buffer_size; | ||
1294 | LIST_HEAD(pages); | ||
1295 | int i, cpu; | ||
1296 | 1319 | ||
1297 | /* | 1320 | /* |
1298 | * Always succeed at resizing a non-existent buffer: | 1321 | * Always succeed at resizing a non-existent buffer: |
@@ -1302,15 +1325,11 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) | |||
1302 | 1325 | ||
1303 | size = DIV_ROUND_UP(size, BUF_PAGE_SIZE); | 1326 | size = DIV_ROUND_UP(size, BUF_PAGE_SIZE); |
1304 | size *= BUF_PAGE_SIZE; | 1327 | size *= BUF_PAGE_SIZE; |
1305 | buffer_size = buffer->pages * BUF_PAGE_SIZE; | ||
1306 | 1328 | ||
1307 | /* we need a minimum of two pages */ | 1329 | /* we need a minimum of two pages */ |
1308 | if (size < BUF_PAGE_SIZE * 2) | 1330 | if (size < BUF_PAGE_SIZE * 2) |
1309 | size = BUF_PAGE_SIZE * 2; | 1331 | size = BUF_PAGE_SIZE * 2; |
1310 | 1332 | ||
1311 | if (size == buffer_size) | ||
1312 | return size; | ||
1313 | |||
1314 | atomic_inc(&buffer->record_disabled); | 1333 | atomic_inc(&buffer->record_disabled); |
1315 | 1334 | ||
1316 | /* Make sure all writers are done with this buffer. */ | 1335 | /* Make sure all writers are done with this buffer. */ |
@@ -1321,68 +1340,56 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) | |||
1321 | 1340 | ||
1322 | nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); | 1341 | nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); |
1323 | 1342 | ||
1324 | if (size < buffer_size) { | 1343 | if (cpu_id == RING_BUFFER_ALL_CPUS) { |
1325 | 1344 | /* calculate the pages to update */ | |
1326 | /* easy case, just free pages */ | ||
1327 | if (RB_WARN_ON(buffer, nr_pages >= buffer->pages)) | ||
1328 | goto out_fail; | ||
1329 | |||
1330 | rm_pages = buffer->pages - nr_pages; | ||
1331 | |||
1332 | for_each_buffer_cpu(buffer, cpu) { | 1345 | for_each_buffer_cpu(buffer, cpu) { |
1333 | cpu_buffer = buffer->buffers[cpu]; | 1346 | cpu_buffer = buffer->buffers[cpu]; |
1334 | rb_remove_pages(cpu_buffer, rm_pages); | ||
1335 | } | ||
1336 | goto out; | ||
1337 | } | ||
1338 | 1347 | ||
1339 | /* | 1348 | cpu_buffer->nr_pages_to_update = nr_pages - |
1340 | * This is a bit more difficult. We only want to add pages | 1349 | cpu_buffer->nr_pages; |
1341 | * when we can allocate enough for all CPUs. We do this | ||
1342 | * by allocating all the pages and storing them on a local | ||
1343 | * link list. If we succeed in our allocation, then we | ||
1344 | * add these pages to the cpu_buffers. Otherwise we just free | ||
1345 | * them all and return -ENOMEM; | ||
1346 | */ | ||
1347 | if (RB_WARN_ON(buffer, nr_pages <= buffer->pages)) | ||
1348 | goto out_fail; | ||
1349 | 1350 | ||
1350 | new_pages = nr_pages - buffer->pages; | 1351 | /* |
1352 | * nothing more to do for removing pages or no update | ||
1353 | */ | ||
1354 | if (cpu_buffer->nr_pages_to_update <= 0) | ||
1355 | continue; | ||
1351 | 1356 | ||
1352 | for_each_buffer_cpu(buffer, cpu) { | ||
1353 | for (i = 0; i < new_pages; i++) { | ||
1354 | struct page *page; | ||
1355 | /* | 1357 | /* |
1356 | * __GFP_NORETRY flag makes sure that the allocation | 1358 | * to add pages, make sure all new pages can be |
1357 | * fails gracefully without invoking oom-killer and | 1359 | * allocated without receiving ENOMEM |
1358 | * the system is not destabilized. | ||
1359 | */ | 1360 | */ |
1360 | bpage = kzalloc_node(ALIGN(sizeof(*bpage), | 1361 | INIT_LIST_HEAD(&cpu_buffer->new_pages); |
1361 | cache_line_size()), | 1362 | if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update, |
1362 | GFP_KERNEL | __GFP_NORETRY, | 1363 | &cpu_buffer->new_pages, cpu)) |
1363 | cpu_to_node(cpu)); | 1364 | /* not enough memory for new pages */ |
1364 | if (!bpage) | 1365 | goto no_mem; |
1365 | goto free_pages; | ||
1366 | list_add(&bpage->list, &pages); | ||
1367 | page = alloc_pages_node(cpu_to_node(cpu), | ||
1368 | GFP_KERNEL | __GFP_NORETRY, 0); | ||
1369 | if (!page) | ||
1370 | goto free_pages; | ||
1371 | bpage->page = page_address(page); | ||
1372 | rb_init_page(bpage->page); | ||
1373 | } | 1366 | } |
1374 | } | ||
1375 | 1367 | ||
1376 | for_each_buffer_cpu(buffer, cpu) { | 1368 | /* wait for all the updates to complete */ |
1377 | cpu_buffer = buffer->buffers[cpu]; | 1369 | for_each_buffer_cpu(buffer, cpu) { |
1378 | rb_insert_pages(cpu_buffer, &pages, new_pages); | 1370 | cpu_buffer = buffer->buffers[cpu]; |
1379 | } | 1371 | if (cpu_buffer->nr_pages_to_update) { |
1372 | update_pages_handler(cpu_buffer); | ||
1373 | } | ||
1374 | } | ||
1375 | } else { | ||
1376 | cpu_buffer = buffer->buffers[cpu_id]; | ||
1377 | if (nr_pages == cpu_buffer->nr_pages) | ||
1378 | goto out; | ||
1380 | 1379 | ||
1381 | if (RB_WARN_ON(buffer, !list_empty(&pages))) | 1380 | cpu_buffer->nr_pages_to_update = nr_pages - |
1382 | goto out_fail; | 1381 | cpu_buffer->nr_pages; |
1382 | |||
1383 | INIT_LIST_HEAD(&cpu_buffer->new_pages); | ||
1384 | if (cpu_buffer->nr_pages_to_update > 0 && | ||
1385 | __rb_allocate_pages(cpu_buffer->nr_pages_to_update, | ||
1386 | &cpu_buffer->new_pages, cpu_id)) | ||
1387 | goto no_mem; | ||
1388 | |||
1389 | update_pages_handler(cpu_buffer); | ||
1390 | } | ||
1383 | 1391 | ||
1384 | out: | 1392 | out: |
1385 | buffer->pages = nr_pages; | ||
1386 | put_online_cpus(); | 1393 | put_online_cpus(); |
1387 | mutex_unlock(&buffer->mutex); | 1394 | mutex_unlock(&buffer->mutex); |
1388 | 1395 | ||
@@ -1390,25 +1397,24 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) | |||
1390 | 1397 | ||
1391 | return size; | 1398 | return size; |
1392 | 1399 | ||
1393 | free_pages: | 1400 | no_mem: |
1394 | list_for_each_entry_safe(bpage, tmp, &pages, list) { | 1401 | for_each_buffer_cpu(buffer, cpu) { |
1395 | list_del_init(&bpage->list); | 1402 | struct buffer_page *bpage, *tmp; |
1396 | free_buffer_page(bpage); | 1403 | cpu_buffer = buffer->buffers[cpu]; |
1404 | /* reset this number regardless */ | ||
1405 | cpu_buffer->nr_pages_to_update = 0; | ||
1406 | if (list_empty(&cpu_buffer->new_pages)) | ||
1407 | continue; | ||
1408 | list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, | ||
1409 | list) { | ||
1410 | list_del_init(&bpage->list); | ||
1411 | free_buffer_page(bpage); | ||
1412 | } | ||
1397 | } | 1413 | } |
1398 | put_online_cpus(); | 1414 | put_online_cpus(); |
1399 | mutex_unlock(&buffer->mutex); | 1415 | mutex_unlock(&buffer->mutex); |
1400 | atomic_dec(&buffer->record_disabled); | 1416 | atomic_dec(&buffer->record_disabled); |
1401 | return -ENOMEM; | 1417 | return -ENOMEM; |
1402 | |||
1403 | /* | ||
1404 | * Something went totally wrong, and we are too paranoid | ||
1405 | * to even clean up the mess. | ||
1406 | */ | ||
1407 | out_fail: | ||
1408 | put_online_cpus(); | ||
1409 | mutex_unlock(&buffer->mutex); | ||
1410 | atomic_dec(&buffer->record_disabled); | ||
1411 | return -1; | ||
1412 | } | 1418 | } |
1413 | EXPORT_SYMBOL_GPL(ring_buffer_resize); | 1419 | EXPORT_SYMBOL_GPL(ring_buffer_resize); |
1414 | 1420 | ||
@@ -1510,7 +1516,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) | |||
1510 | * assign the commit to the tail. | 1516 | * assign the commit to the tail. |
1511 | */ | 1517 | */ |
1512 | again: | 1518 | again: |
1513 | max_count = cpu_buffer->buffer->pages * 100; | 1519 | max_count = cpu_buffer->nr_pages * 100; |
1514 | 1520 | ||
1515 | while (cpu_buffer->commit_page != cpu_buffer->tail_page) { | 1521 | while (cpu_buffer->commit_page != cpu_buffer->tail_page) { |
1516 | if (RB_WARN_ON(cpu_buffer, !(--max_count))) | 1522 | if (RB_WARN_ON(cpu_buffer, !(--max_count))) |
@@ -3588,9 +3594,18 @@ EXPORT_SYMBOL_GPL(ring_buffer_read); | |||
3588 | * ring_buffer_size - return the size of the ring buffer (in bytes) | 3594 | * ring_buffer_size - return the size of the ring buffer (in bytes) |
3589 | * @buffer: The ring buffer. | 3595 | * @buffer: The ring buffer. |
3590 | */ | 3596 | */ |
3591 | unsigned long ring_buffer_size(struct ring_buffer *buffer) | 3597 | unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu) |
3592 | { | 3598 | { |
3593 | return BUF_PAGE_SIZE * buffer->pages; | 3599 | /* |
3600 | * Earlier, this method returned | ||
3601 | * BUF_PAGE_SIZE * buffer->nr_pages | ||
3602 | * Since the nr_pages field is now removed, we have converted this to | ||
3603 | * return the per cpu buffer value. | ||
3604 | */ | ||
3605 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | ||
3606 | return 0; | ||
3607 | |||
3608 | return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages; | ||
3594 | } | 3609 | } |
3595 | EXPORT_SYMBOL_GPL(ring_buffer_size); | 3610 | EXPORT_SYMBOL_GPL(ring_buffer_size); |
3596 | 3611 | ||
@@ -3765,8 +3780,11 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a, | |||
3765 | !cpumask_test_cpu(cpu, buffer_b->cpumask)) | 3780 | !cpumask_test_cpu(cpu, buffer_b->cpumask)) |
3766 | goto out; | 3781 | goto out; |
3767 | 3782 | ||
3783 | cpu_buffer_a = buffer_a->buffers[cpu]; | ||
3784 | cpu_buffer_b = buffer_b->buffers[cpu]; | ||
3785 | |||
3768 | /* At least make sure the two buffers are somewhat the same */ | 3786 | /* At least make sure the two buffers are somewhat the same */ |
3769 | if (buffer_a->pages != buffer_b->pages) | 3787 | if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages) |
3770 | goto out; | 3788 | goto out; |
3771 | 3789 | ||
3772 | ret = -EAGAIN; | 3790 | ret = -EAGAIN; |
@@ -3780,9 +3798,6 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a, | |||
3780 | if (atomic_read(&buffer_b->record_disabled)) | 3798 | if (atomic_read(&buffer_b->record_disabled)) |
3781 | goto out; | 3799 | goto out; |
3782 | 3800 | ||
3783 | cpu_buffer_a = buffer_a->buffers[cpu]; | ||
3784 | cpu_buffer_b = buffer_b->buffers[cpu]; | ||
3785 | |||
3786 | if (atomic_read(&cpu_buffer_a->record_disabled)) | 3801 | if (atomic_read(&cpu_buffer_a->record_disabled)) |
3787 | goto out; | 3802 | goto out; |
3788 | 3803 | ||
@@ -4071,6 +4086,8 @@ static int rb_cpu_notify(struct notifier_block *self, | |||
4071 | struct ring_buffer *buffer = | 4086 | struct ring_buffer *buffer = |
4072 | container_of(self, struct ring_buffer, cpu_notify); | 4087 | container_of(self, struct ring_buffer, cpu_notify); |
4073 | long cpu = (long)hcpu; | 4088 | long cpu = (long)hcpu; |
4089 | int cpu_i, nr_pages_same; | ||
4090 | unsigned int nr_pages; | ||
4074 | 4091 | ||
4075 | switch (action) { | 4092 | switch (action) { |
4076 | case CPU_UP_PREPARE: | 4093 | case CPU_UP_PREPARE: |
@@ -4078,8 +4095,23 @@ static int rb_cpu_notify(struct notifier_block *self, | |||
4078 | if (cpumask_test_cpu(cpu, buffer->cpumask)) | 4095 | if (cpumask_test_cpu(cpu, buffer->cpumask)) |
4079 | return NOTIFY_OK; | 4096 | return NOTIFY_OK; |
4080 | 4097 | ||
4098 | nr_pages = 0; | ||
4099 | nr_pages_same = 1; | ||
4100 | /* check if all cpu sizes are same */ | ||
4101 | for_each_buffer_cpu(buffer, cpu_i) { | ||
4102 | /* fill in the size from first enabled cpu */ | ||
4103 | if (nr_pages == 0) | ||
4104 | nr_pages = buffer->buffers[cpu_i]->nr_pages; | ||
4105 | if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { | ||
4106 | nr_pages_same = 0; | ||
4107 | break; | ||
4108 | } | ||
4109 | } | ||
4110 | /* allocate minimum pages, user can later expand it */ | ||
4111 | if (!nr_pages_same) | ||
4112 | nr_pages = 2; | ||
4081 | buffer->buffers[cpu] = | 4113 | buffer->buffers[cpu] = |
4082 | rb_allocate_cpu_buffer(buffer, cpu); | 4114 | rb_allocate_cpu_buffer(buffer, nr_pages, cpu); |
4083 | if (!buffer->buffers[cpu]) { | 4115 | if (!buffer->buffers[cpu]) { |
4084 | WARN(1, "failed to allocate ring buffer on CPU %ld\n", | 4116 | WARN(1, "failed to allocate ring buffer on CPU %ld\n", |
4085 | cpu); | 4117 | cpu); |
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 2a22255c101..509e8615f50 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
@@ -87,18 +87,6 @@ static int tracing_disabled = 1; | |||
87 | 87 | ||
88 | DEFINE_PER_CPU(int, ftrace_cpu_disabled); | 88 | DEFINE_PER_CPU(int, ftrace_cpu_disabled); |
89 | 89 | ||
90 | static inline void ftrace_disable_cpu(void) | ||
91 | { | ||
92 | preempt_disable(); | ||
93 | __this_cpu_inc(ftrace_cpu_disabled); | ||
94 | } | ||
95 | |||
96 | static inline void ftrace_enable_cpu(void) | ||
97 | { | ||
98 | __this_cpu_dec(ftrace_cpu_disabled); | ||
99 | preempt_enable(); | ||
100 | } | ||
101 | |||
102 | cpumask_var_t __read_mostly tracing_buffer_mask; | 90 | cpumask_var_t __read_mostly tracing_buffer_mask; |
103 | 91 | ||
104 | /* | 92 | /* |
@@ -629,7 +617,6 @@ ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt) | |||
629 | static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) | 617 | static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) |
630 | { | 618 | { |
631 | int len; | 619 | int len; |
632 | void *ret; | ||
633 | 620 | ||
634 | if (s->len <= s->readpos) | 621 | if (s->len <= s->readpos) |
635 | return -EBUSY; | 622 | return -EBUSY; |
@@ -637,9 +624,7 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) | |||
637 | len = s->len - s->readpos; | 624 | len = s->len - s->readpos; |
638 | if (cnt > len) | 625 | if (cnt > len) |
639 | cnt = len; | 626 | cnt = len; |
640 | ret = memcpy(buf, s->buffer + s->readpos, cnt); | 627 | memcpy(buf, s->buffer + s->readpos, cnt); |
641 | if (!ret) | ||
642 | return -EFAULT; | ||
643 | 628 | ||
644 | s->readpos += cnt; | 629 | s->readpos += cnt; |
645 | return cnt; | 630 | return cnt; |
@@ -751,8 +736,6 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) | |||
751 | 736 | ||
752 | arch_spin_lock(&ftrace_max_lock); | 737 | arch_spin_lock(&ftrace_max_lock); |
753 | 738 | ||
754 | ftrace_disable_cpu(); | ||
755 | |||
756 | ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu); | 739 | ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu); |
757 | 740 | ||
758 | if (ret == -EBUSY) { | 741 | if (ret == -EBUSY) { |
@@ -766,8 +749,6 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) | |||
766 | "Failed to swap buffers due to commit in progress\n"); | 749 | "Failed to swap buffers due to commit in progress\n"); |
767 | } | 750 | } |
768 | 751 | ||
769 | ftrace_enable_cpu(); | ||
770 | |||
771 | WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); | 752 | WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); |
772 | 753 | ||
773 | __update_max_tr(tr, tsk, cpu); | 754 | __update_max_tr(tr, tsk, cpu); |
@@ -841,7 +822,8 @@ __acquires(kernel_lock) | |||
841 | 822 | ||
842 | /* If we expanded the buffers, make sure the max is expanded too */ | 823 | /* If we expanded the buffers, make sure the max is expanded too */ |
843 | if (ring_buffer_expanded && type->use_max_tr) | 824 | if (ring_buffer_expanded && type->use_max_tr) |
844 | ring_buffer_resize(max_tr.buffer, trace_buf_size); | 825 | ring_buffer_resize(max_tr.buffer, trace_buf_size, |
826 | RING_BUFFER_ALL_CPUS); | ||
845 | 827 | ||
846 | /* the test is responsible for initializing and enabling */ | 828 | /* the test is responsible for initializing and enabling */ |
847 | pr_info("Testing tracer %s: ", type->name); | 829 | pr_info("Testing tracer %s: ", type->name); |
@@ -857,7 +839,8 @@ __acquires(kernel_lock) | |||
857 | 839 | ||
858 | /* Shrink the max buffer again */ | 840 | /* Shrink the max buffer again */ |
859 | if (ring_buffer_expanded && type->use_max_tr) | 841 | if (ring_buffer_expanded && type->use_max_tr) |
860 | ring_buffer_resize(max_tr.buffer, 1); | 842 | ring_buffer_resize(max_tr.buffer, 1, |
843 | RING_BUFFER_ALL_CPUS); | ||
861 | 844 | ||
862 | printk(KERN_CONT "PASSED\n"); | 845 | printk(KERN_CONT "PASSED\n"); |
863 | } | 846 | } |
@@ -917,13 +900,6 @@ out: | |||
917 | mutex_unlock(&trace_types_lock); | 900 | mutex_unlock(&trace_types_lock); |
918 | } | 901 | } |
919 | 902 | ||
920 | static void __tracing_reset(struct ring_buffer *buffer, int cpu) | ||
921 | { | ||
922 | ftrace_disable_cpu(); | ||
923 | ring_buffer_reset_cpu(buffer, cpu); | ||
924 | ftrace_enable_cpu(); | ||
925 | } | ||
926 | |||
927 | void tracing_reset(struct trace_array *tr, int cpu) | 903 | void tracing_reset(struct trace_array *tr, int cpu) |
928 | { | 904 | { |
929 | struct ring_buffer *buffer = tr->buffer; | 905 | struct ring_buffer *buffer = tr->buffer; |
@@ -932,7 +908,7 @@ void tracing_reset(struct trace_array *tr, int cpu) | |||
932 | 908 | ||
933 | /* Make sure all commits have finished */ | 909 | /* Make sure all commits have finished */ |
934 | synchronize_sched(); | 910 | synchronize_sched(); |
935 | __tracing_reset(buffer, cpu); | 911 | ring_buffer_reset_cpu(buffer, cpu); |
936 | 912 | ||
937 | ring_buffer_record_enable(buffer); | 913 | ring_buffer_record_enable(buffer); |
938 | } | 914 | } |
@@ -950,7 +926,7 @@ void tracing_reset_online_cpus(struct trace_array *tr) | |||
950 | tr->time_start = ftrace_now(tr->cpu); | 926 | tr->time_start = ftrace_now(tr->cpu); |
951 | 927 | ||
952 | for_each_online_cpu(cpu) | 928 | for_each_online_cpu(cpu) |
953 | __tracing_reset(buffer, cpu); | 929 | ring_buffer_reset_cpu(buffer, cpu); |
954 | 930 | ||
955 | ring_buffer_record_enable(buffer); | 931 | ring_buffer_record_enable(buffer); |
956 | } | 932 | } |
@@ -1498,25 +1474,119 @@ static void __trace_userstack(struct trace_array *tr, unsigned long flags) | |||
1498 | 1474 | ||
1499 | #endif /* CONFIG_STACKTRACE */ | 1475 | #endif /* CONFIG_STACKTRACE */ |
1500 | 1476 | ||
1477 | /* created for use with alloc_percpu */ | ||
1478 | struct trace_buffer_struct { | ||
1479 | char buffer[TRACE_BUF_SIZE]; | ||
1480 | }; | ||
1481 | |||
1482 | static struct trace_buffer_struct *trace_percpu_buffer; | ||
1483 | static struct trace_buffer_struct *trace_percpu_sirq_buffer; | ||
1484 | static struct trace_buffer_struct *trace_percpu_irq_buffer; | ||
1485 | static struct trace_buffer_struct *trace_percpu_nmi_buffer; | ||
1486 | |||
1487 | /* | ||
1488 | * The buffer used is dependent on the context. There is a per cpu | ||
1489 | * buffer for normal context, softirq contex, hard irq context and | ||
1490 | * for NMI context. Thise allows for lockless recording. | ||
1491 | * | ||
1492 | * Note, if the buffers failed to be allocated, then this returns NULL | ||
1493 | */ | ||
1494 | static char *get_trace_buf(void) | ||
1495 | { | ||
1496 | struct trace_buffer_struct *percpu_buffer; | ||
1497 | struct trace_buffer_struct *buffer; | ||
1498 | |||
1499 | /* | ||
1500 | * If we have allocated per cpu buffers, then we do not | ||
1501 | * need to do any locking. | ||
1502 | */ | ||
1503 | if (in_nmi()) | ||
1504 | percpu_buffer = trace_percpu_nmi_buffer; | ||
1505 | else if (in_irq()) | ||
1506 | percpu_buffer = trace_percpu_irq_buffer; | ||
1507 | else if (in_softirq()) | ||
1508 | percpu_buffer = trace_percpu_sirq_buffer; | ||
1509 | else | ||
1510 | percpu_buffer = trace_percpu_buffer; | ||
1511 | |||
1512 | if (!percpu_buffer) | ||
1513 | return NULL; | ||
1514 | |||
1515 | buffer = per_cpu_ptr(percpu_buffer, smp_processor_id()); | ||
1516 | |||
1517 | return buffer->buffer; | ||
1518 | } | ||
1519 | |||
1520 | static int alloc_percpu_trace_buffer(void) | ||
1521 | { | ||
1522 | struct trace_buffer_struct *buffers; | ||
1523 | struct trace_buffer_struct *sirq_buffers; | ||
1524 | struct trace_buffer_struct *irq_buffers; | ||
1525 | struct trace_buffer_struct *nmi_buffers; | ||
1526 | |||
1527 | buffers = alloc_percpu(struct trace_buffer_struct); | ||
1528 | if (!buffers) | ||
1529 | goto err_warn; | ||
1530 | |||
1531 | sirq_buffers = alloc_percpu(struct trace_buffer_struct); | ||
1532 | if (!sirq_buffers) | ||
1533 | goto err_sirq; | ||
1534 | |||
1535 | irq_buffers = alloc_percpu(struct trace_buffer_struct); | ||
1536 | if (!irq_buffers) | ||
1537 | goto err_irq; | ||
1538 | |||
1539 | nmi_buffers = alloc_percpu(struct trace_buffer_struct); | ||
1540 | if (!nmi_buffers) | ||
1541 | goto err_nmi; | ||
1542 | |||
1543 | trace_percpu_buffer = buffers; | ||
1544 | trace_percpu_sirq_buffer = sirq_buffers; | ||
1545 | trace_percpu_irq_buffer = irq_buffers; | ||
1546 | trace_percpu_nmi_buffer = nmi_buffers; | ||
1547 | |||
1548 | return 0; | ||
1549 | |||
1550 | err_nmi: | ||
1551 | free_percpu(irq_buffers); | ||
1552 | err_irq: | ||
1553 | free_percpu(sirq_buffers); | ||
1554 | err_sirq: | ||
1555 | free_percpu(buffers); | ||
1556 | err_warn: | ||
1557 | WARN(1, "Could not allocate percpu trace_printk buffer"); | ||
1558 | return -ENOMEM; | ||
1559 | } | ||
1560 | |||
1561 | void trace_printk_init_buffers(void) | ||
1562 | { | ||
1563 | static int buffers_allocated; | ||
1564 | |||
1565 | if (buffers_allocated) | ||
1566 | return; | ||
1567 | |||
1568 | if (alloc_percpu_trace_buffer()) | ||
1569 | return; | ||
1570 | |||
1571 | pr_info("ftrace: Allocated trace_printk buffers\n"); | ||
1572 | |||
1573 | buffers_allocated = 1; | ||
1574 | } | ||
1575 | |||
1501 | /** | 1576 | /** |
1502 | * trace_vbprintk - write binary msg to tracing buffer | 1577 | * trace_vbprintk - write binary msg to tracing buffer |
1503 | * | 1578 | * |
1504 | */ | 1579 | */ |
1505 | int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) | 1580 | int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) |
1506 | { | 1581 | { |
1507 | static arch_spinlock_t trace_buf_lock = | ||
1508 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; | ||
1509 | static u32 trace_buf[TRACE_BUF_SIZE]; | ||
1510 | |||
1511 | struct ftrace_event_call *call = &event_bprint; | 1582 | struct ftrace_event_call *call = &event_bprint; |
1512 | struct ring_buffer_event *event; | 1583 | struct ring_buffer_event *event; |
1513 | struct ring_buffer *buffer; | 1584 | struct ring_buffer *buffer; |
1514 | struct trace_array *tr = &global_trace; | 1585 | struct trace_array *tr = &global_trace; |
1515 | struct trace_array_cpu *data; | ||
1516 | struct bprint_entry *entry; | 1586 | struct bprint_entry *entry; |
1517 | unsigned long flags; | 1587 | unsigned long flags; |
1518 | int disable; | 1588 | char *tbuffer; |
1519 | int cpu, len = 0, size, pc; | 1589 | int len = 0, size, pc; |
1520 | 1590 | ||
1521 | if (unlikely(tracing_selftest_running || tracing_disabled)) | 1591 | if (unlikely(tracing_selftest_running || tracing_disabled)) |
1522 | return 0; | 1592 | return 0; |
@@ -1526,43 +1596,36 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) | |||
1526 | 1596 | ||
1527 | pc = preempt_count(); | 1597 | pc = preempt_count(); |
1528 | preempt_disable_notrace(); | 1598 | preempt_disable_notrace(); |
1529 | cpu = raw_smp_processor_id(); | ||
1530 | data = tr->data[cpu]; | ||
1531 | 1599 | ||
1532 | disable = atomic_inc_return(&data->disabled); | 1600 | tbuffer = get_trace_buf(); |
1533 | if (unlikely(disable != 1)) | 1601 | if (!tbuffer) { |
1602 | len = 0; | ||
1534 | goto out; | 1603 | goto out; |
1604 | } | ||
1535 | 1605 | ||
1536 | /* Lockdep uses trace_printk for lock tracing */ | 1606 | len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); |
1537 | local_irq_save(flags); | ||
1538 | arch_spin_lock(&trace_buf_lock); | ||
1539 | len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args); | ||
1540 | 1607 | ||
1541 | if (len > TRACE_BUF_SIZE || len < 0) | 1608 | if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) |
1542 | goto out_unlock; | 1609 | goto out; |
1543 | 1610 | ||
1611 | local_save_flags(flags); | ||
1544 | size = sizeof(*entry) + sizeof(u32) * len; | 1612 | size = sizeof(*entry) + sizeof(u32) * len; |
1545 | buffer = tr->buffer; | 1613 | buffer = tr->buffer; |
1546 | event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, | 1614 | event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, |
1547 | flags, pc); | 1615 | flags, pc); |
1548 | if (!event) | 1616 | if (!event) |
1549 | goto out_unlock; | 1617 | goto out; |
1550 | entry = ring_buffer_event_data(event); | 1618 | entry = ring_buffer_event_data(event); |
1551 | entry->ip = ip; | 1619 | entry->ip = ip; |
1552 | entry->fmt = fmt; | 1620 | entry->fmt = fmt; |
1553 | 1621 | ||
1554 | memcpy(entry->buf, trace_buf, sizeof(u32) * len); | 1622 | memcpy(entry->buf, tbuffer, sizeof(u32) * len); |
1555 | if (!filter_check_discard(call, entry, buffer, event)) { | 1623 | if (!filter_check_discard(call, entry, buffer, event)) { |
1556 | ring_buffer_unlock_commit(buffer, event); | 1624 | ring_buffer_unlock_commit(buffer, event); |
1557 | ftrace_trace_stack(buffer, flags, 6, pc); | 1625 | ftrace_trace_stack(buffer, flags, 6, pc); |
1558 | } | 1626 | } |
1559 | 1627 | ||
1560 | out_unlock: | ||
1561 | arch_spin_unlock(&trace_buf_lock); | ||
1562 | local_irq_restore(flags); | ||
1563 | |||
1564 | out: | 1628 | out: |
1565 | atomic_dec_return(&data->disabled); | ||
1566 | preempt_enable_notrace(); | 1629 | preempt_enable_notrace(); |
1567 | unpause_graph_tracing(); | 1630 | unpause_graph_tracing(); |
1568 | 1631 | ||
@@ -1588,58 +1651,53 @@ int trace_array_printk(struct trace_array *tr, | |||
1588 | int trace_array_vprintk(struct trace_array *tr, | 1651 | int trace_array_vprintk(struct trace_array *tr, |
1589 | unsigned long ip, const char *fmt, va_list args) | 1652 | unsigned long ip, const char *fmt, va_list args) |
1590 | { | 1653 | { |
1591 | static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED; | ||
1592 | static char trace_buf[TRACE_BUF_SIZE]; | ||
1593 | |||
1594 | struct ftrace_event_call *call = &event_print; | 1654 | struct ftrace_event_call *call = &event_print; |
1595 | struct ring_buffer_event *event; | 1655 | struct ring_buffer_event *event; |
1596 | struct ring_buffer *buffer; | 1656 | struct ring_buffer *buffer; |
1597 | struct trace_array_cpu *data; | 1657 | int len = 0, size, pc; |
1598 | int cpu, len = 0, size, pc; | ||
1599 | struct print_entry *entry; | 1658 | struct print_entry *entry; |
1600 | unsigned long irq_flags; | 1659 | unsigned long flags; |
1601 | int disable; | 1660 | char *tbuffer; |
1602 | 1661 | ||
1603 | if (tracing_disabled || tracing_selftest_running) | 1662 | if (tracing_disabled || tracing_selftest_running) |
1604 | return 0; | 1663 | return 0; |
1605 | 1664 | ||
1665 | /* Don't pollute graph traces with trace_vprintk internals */ | ||
1666 | pause_graph_tracing(); | ||
1667 | |||
1606 | pc = preempt_count(); | 1668 | pc = preempt_count(); |
1607 | preempt_disable_notrace(); | 1669 | preempt_disable_notrace(); |
1608 | cpu = raw_smp_processor_id(); | ||
1609 | data = tr->data[cpu]; | ||
1610 | 1670 | ||
1611 | disable = atomic_inc_return(&data->disabled); | 1671 | |
1612 | if (unlikely(disable != 1)) | 1672 | tbuffer = get_trace_buf(); |
1673 | if (!tbuffer) { | ||
1674 | len = 0; | ||
1613 | goto out; | 1675 | goto out; |
1676 | } | ||
1614 | 1677 | ||
1615 | pause_graph_tracing(); | 1678 | len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); |
1616 | raw_local_irq_save(irq_flags); | 1679 | if (len > TRACE_BUF_SIZE) |
1617 | arch_spin_lock(&trace_buf_lock); | 1680 | goto out; |
1618 | len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args); | ||
1619 | 1681 | ||
1682 | local_save_flags(flags); | ||
1620 | size = sizeof(*entry) + len + 1; | 1683 | size = sizeof(*entry) + len + 1; |
1621 | buffer = tr->buffer; | 1684 | buffer = tr->buffer; |
1622 | event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, | 1685 | event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, |
1623 | irq_flags, pc); | 1686 | flags, pc); |
1624 | if (!event) | 1687 | if (!event) |
1625 | goto out_unlock; | 1688 | goto out; |
1626 | entry = ring_buffer_event_data(event); | 1689 | entry = ring_buffer_event_data(event); |
1627 | entry->ip = ip; | 1690 | entry->ip = ip; |
1628 | 1691 | ||
1629 | memcpy(&entry->buf, trace_buf, len); | 1692 | memcpy(&entry->buf, tbuffer, len); |
1630 | entry->buf[len] = '\0'; | 1693 | entry->buf[len] = '\0'; |
1631 | if (!filter_check_discard(call, entry, buffer, event)) { | 1694 | if (!filter_check_discard(call, entry, buffer, event)) { |
1632 | ring_buffer_unlock_commit(buffer, event); | 1695 | ring_buffer_unlock_commit(buffer, event); |
1633 | ftrace_trace_stack(buffer, irq_flags, 6, pc); | 1696 | ftrace_trace_stack(buffer, flags, 6, pc); |
1634 | } | 1697 | } |
1635 | |||
1636 | out_unlock: | ||
1637 | arch_spin_unlock(&trace_buf_lock); | ||
1638 | raw_local_irq_restore(irq_flags); | ||
1639 | unpause_graph_tracing(); | ||
1640 | out: | 1698 | out: |
1641 | atomic_dec_return(&data->disabled); | ||
1642 | preempt_enable_notrace(); | 1699 | preempt_enable_notrace(); |
1700 | unpause_graph_tracing(); | ||
1643 | 1701 | ||
1644 | return len; | 1702 | return len; |
1645 | } | 1703 | } |
@@ -1652,14 +1710,9 @@ EXPORT_SYMBOL_GPL(trace_vprintk); | |||
1652 | 1710 | ||
1653 | static void trace_iterator_increment(struct trace_iterator *iter) | 1711 | static void trace_iterator_increment(struct trace_iterator *iter) |
1654 | { | 1712 | { |
1655 | /* Don't allow ftrace to trace into the ring buffers */ | ||
1656 | ftrace_disable_cpu(); | ||
1657 | |||
1658 | iter->idx++; | 1713 | iter->idx++; |
1659 | if (iter->buffer_iter[iter->cpu]) | 1714 | if (iter->buffer_iter[iter->cpu]) |
1660 | ring_buffer_read(iter->buffer_iter[iter->cpu], NULL); | 1715 | ring_buffer_read(iter->buffer_iter[iter->cpu], NULL); |
1661 | |||
1662 | ftrace_enable_cpu(); | ||
1663 | } | 1716 | } |
1664 | 1717 | ||
1665 | static struct trace_entry * | 1718 | static struct trace_entry * |
@@ -1669,17 +1722,12 @@ peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, | |||
1669 | struct ring_buffer_event *event; | 1722 | struct ring_buffer_event *event; |
1670 | struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu]; | 1723 | struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu]; |
1671 | 1724 | ||
1672 | /* Don't allow ftrace to trace into the ring buffers */ | ||
1673 | ftrace_disable_cpu(); | ||
1674 | |||
1675 | if (buf_iter) | 1725 | if (buf_iter) |
1676 | event = ring_buffer_iter_peek(buf_iter, ts); | 1726 | event = ring_buffer_iter_peek(buf_iter, ts); |
1677 | else | 1727 | else |
1678 | event = ring_buffer_peek(iter->tr->buffer, cpu, ts, | 1728 | event = ring_buffer_peek(iter->tr->buffer, cpu, ts, |
1679 | lost_events); | 1729 | lost_events); |
1680 | 1730 | ||
1681 | ftrace_enable_cpu(); | ||
1682 | |||
1683 | if (event) { | 1731 | if (event) { |
1684 | iter->ent_size = ring_buffer_event_length(event); | 1732 | iter->ent_size = ring_buffer_event_length(event); |
1685 | return ring_buffer_event_data(event); | 1733 | return ring_buffer_event_data(event); |
@@ -1769,11 +1817,8 @@ void *trace_find_next_entry_inc(struct trace_iterator *iter) | |||
1769 | 1817 | ||
1770 | static void trace_consume(struct trace_iterator *iter) | 1818 | static void trace_consume(struct trace_iterator *iter) |
1771 | { | 1819 | { |
1772 | /* Don't allow ftrace to trace into the ring buffers */ | ||
1773 | ftrace_disable_cpu(); | ||
1774 | ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts, | 1820 | ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts, |
1775 | &iter->lost_events); | 1821 | &iter->lost_events); |
1776 | ftrace_enable_cpu(); | ||
1777 | } | 1822 | } |
1778 | 1823 | ||
1779 | static void *s_next(struct seq_file *m, void *v, loff_t *pos) | 1824 | static void *s_next(struct seq_file *m, void *v, loff_t *pos) |
@@ -1862,16 +1907,12 @@ static void *s_start(struct seq_file *m, loff_t *pos) | |||
1862 | iter->cpu = 0; | 1907 | iter->cpu = 0; |
1863 | iter->idx = -1; | 1908 | iter->idx = -1; |
1864 | 1909 | ||
1865 | ftrace_disable_cpu(); | ||
1866 | |||
1867 | if (cpu_file == TRACE_PIPE_ALL_CPU) { | 1910 | if (cpu_file == TRACE_PIPE_ALL_CPU) { |
1868 | for_each_tracing_cpu(cpu) | 1911 | for_each_tracing_cpu(cpu) |
1869 | tracing_iter_reset(iter, cpu); | 1912 | tracing_iter_reset(iter, cpu); |
1870 | } else | 1913 | } else |
1871 | tracing_iter_reset(iter, cpu_file); | 1914 | tracing_iter_reset(iter, cpu_file); |
1872 | 1915 | ||
1873 | ftrace_enable_cpu(); | ||
1874 | |||
1875 | iter->leftover = 0; | 1916 | iter->leftover = 0; |
1876 | for (p = iter; p && l < *pos; p = s_next(m, p, &l)) | 1917 | for (p = iter; p && l < *pos; p = s_next(m, p, &l)) |
1877 | ; | 1918 | ; |
@@ -2332,15 +2373,13 @@ static struct trace_iterator * | |||
2332 | __tracing_open(struct inode *inode, struct file *file) | 2373 | __tracing_open(struct inode *inode, struct file *file) |
2333 | { | 2374 | { |
2334 | long cpu_file = (long) inode->i_private; | 2375 | long cpu_file = (long) inode->i_private; |
2335 | void *fail_ret = ERR_PTR(-ENOMEM); | ||
2336 | struct trace_iterator *iter; | 2376 | struct trace_iterator *iter; |
2337 | struct seq_file *m; | 2377 | int cpu; |
2338 | int cpu, ret; | ||
2339 | 2378 | ||
2340 | if (tracing_disabled) | 2379 | if (tracing_disabled) |
2341 | return ERR_PTR(-ENODEV); | 2380 | return ERR_PTR(-ENODEV); |
2342 | 2381 | ||
2343 | iter = kzalloc(sizeof(*iter), GFP_KERNEL); | 2382 | iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); |
2344 | if (!iter) | 2383 | if (!iter) |
2345 | return ERR_PTR(-ENOMEM); | 2384 | return ERR_PTR(-ENOMEM); |
2346 | 2385 | ||
@@ -2397,32 +2436,15 @@ __tracing_open(struct inode *inode, struct file *file) | |||
2397 | tracing_iter_reset(iter, cpu); | 2436 | tracing_iter_reset(iter, cpu); |
2398 | } | 2437 | } |
2399 | 2438 | ||
2400 | ret = seq_open(file, &tracer_seq_ops); | ||
2401 | if (ret < 0) { | ||
2402 | fail_ret = ERR_PTR(ret); | ||
2403 | goto fail_buffer; | ||
2404 | } | ||
2405 | |||
2406 | m = file->private_data; | ||
2407 | m->private = iter; | ||
2408 | |||
2409 | mutex_unlock(&trace_types_lock); | 2439 | mutex_unlock(&trace_types_lock); |
2410 | 2440 | ||
2411 | return iter; | 2441 | return iter; |
2412 | 2442 | ||
2413 | fail_buffer: | ||
2414 | for_each_tracing_cpu(cpu) { | ||
2415 | if (iter->buffer_iter[cpu]) | ||
2416 | ring_buffer_read_finish(iter->buffer_iter[cpu]); | ||
2417 | } | ||
2418 | free_cpumask_var(iter->started); | ||
2419 | tracing_start(); | ||
2420 | fail: | 2443 | fail: |
2421 | mutex_unlock(&trace_types_lock); | 2444 | mutex_unlock(&trace_types_lock); |
2422 | kfree(iter->trace); | 2445 | kfree(iter->trace); |
2423 | kfree(iter); | 2446 | seq_release_private(inode, file); |
2424 | 2447 | return ERR_PTR(-ENOMEM); | |
2425 | return fail_ret; | ||
2426 | } | 2448 | } |
2427 | 2449 | ||
2428 | int tracing_open_generic(struct inode *inode, struct file *filp) | 2450 | int tracing_open_generic(struct inode *inode, struct file *filp) |
@@ -2458,11 +2480,10 @@ static int tracing_release(struct inode *inode, struct file *file) | |||
2458 | tracing_start(); | 2480 | tracing_start(); |
2459 | mutex_unlock(&trace_types_lock); | 2481 | mutex_unlock(&trace_types_lock); |
2460 | 2482 | ||
2461 | seq_release(inode, file); | ||
2462 | mutex_destroy(&iter->mutex); | 2483 | mutex_destroy(&iter->mutex); |
2463 | free_cpumask_var(iter->started); | 2484 | free_cpumask_var(iter->started); |
2464 | kfree(iter->trace); | 2485 | kfree(iter->trace); |
2465 | kfree(iter); | 2486 | seq_release_private(inode, file); |
2466 | return 0; | 2487 | return 0; |
2467 | } | 2488 | } |
2468 | 2489 | ||
@@ -2974,7 +2995,14 @@ int tracer_init(struct tracer *t, struct trace_array *tr) | |||
2974 | return t->init(tr); | 2995 | return t->init(tr); |
2975 | } | 2996 | } |
2976 | 2997 | ||
2977 | static int __tracing_resize_ring_buffer(unsigned long size) | 2998 | static void set_buffer_entries(struct trace_array *tr, unsigned long val) |
2999 | { | ||
3000 | int cpu; | ||
3001 | for_each_tracing_cpu(cpu) | ||
3002 | tr->data[cpu]->entries = val; | ||
3003 | } | ||
3004 | |||
3005 | static int __tracing_resize_ring_buffer(unsigned long size, int cpu) | ||
2978 | { | 3006 | { |
2979 | int ret; | 3007 | int ret; |
2980 | 3008 | ||
@@ -2985,19 +3013,32 @@ static int __tracing_resize_ring_buffer(unsigned long size) | |||
2985 | */ | 3013 | */ |
2986 | ring_buffer_expanded = 1; | 3014 | ring_buffer_expanded = 1; |
2987 | 3015 | ||
2988 | ret = ring_buffer_resize(global_trace.buffer, size); | 3016 | ret = ring_buffer_resize(global_trace.buffer, size, cpu); |
2989 | if (ret < 0) | 3017 | if (ret < 0) |
2990 | return ret; | 3018 | return ret; |
2991 | 3019 | ||
2992 | if (!current_trace->use_max_tr) | 3020 | if (!current_trace->use_max_tr) |
2993 | goto out; | 3021 | goto out; |
2994 | 3022 | ||
2995 | ret = ring_buffer_resize(max_tr.buffer, size); | 3023 | ret = ring_buffer_resize(max_tr.buffer, size, cpu); |
2996 | if (ret < 0) { | 3024 | if (ret < 0) { |
2997 | int r; | 3025 | int r = 0; |
3026 | |||
3027 | if (cpu == RING_BUFFER_ALL_CPUS) { | ||
3028 | int i; | ||
3029 | for_each_tracing_cpu(i) { | ||
3030 | r = ring_buffer_resize(global_trace.buffer, | ||
3031 | global_trace.data[i]->entries, | ||
3032 | i); | ||
3033 | if (r < 0) | ||
3034 | break; | ||
3035 | } | ||
3036 | } else { | ||
3037 | r = ring_buffer_resize(global_trace.buffer, | ||
3038 | global_trace.data[cpu]->entries, | ||
3039 | cpu); | ||
3040 | } | ||
2998 | 3041 | ||
2999 | r = ring_buffer_resize(global_trace.buffer, | ||
3000 | global_trace.entries); | ||
3001 | if (r < 0) { | 3042 | if (r < 0) { |
3002 | /* | 3043 | /* |
3003 | * AARGH! We are left with different | 3044 | * AARGH! We are left with different |
@@ -3019,14 +3060,21 @@ static int __tracing_resize_ring_buffer(unsigned long size) | |||
3019 | return ret; | 3060 | return ret; |
3020 | } | 3061 | } |
3021 | 3062 | ||
3022 | max_tr.entries = size; | 3063 | if (cpu == RING_BUFFER_ALL_CPUS) |
3064 | set_buffer_entries(&max_tr, size); | ||
3065 | else | ||
3066 | max_tr.data[cpu]->entries = size; | ||
3067 | |||
3023 | out: | 3068 | out: |
3024 | global_trace.entries = size; | 3069 | if (cpu == RING_BUFFER_ALL_CPUS) |
3070 | set_buffer_entries(&global_trace, size); | ||
3071 | else | ||
3072 | global_trace.data[cpu]->entries = size; | ||
3025 | 3073 | ||
3026 | return ret; | 3074 | return ret; |
3027 | } | 3075 | } |
3028 | 3076 | ||
3029 | static ssize_t tracing_resize_ring_buffer(unsigned long size) | 3077 | static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id) |
3030 | { | 3078 | { |
3031 | int cpu, ret = size; | 3079 | int cpu, ret = size; |
3032 | 3080 | ||
@@ -3042,12 +3090,19 @@ static ssize_t tracing_resize_ring_buffer(unsigned long size) | |||
3042 | atomic_inc(&max_tr.data[cpu]->disabled); | 3090 | atomic_inc(&max_tr.data[cpu]->disabled); |
3043 | } | 3091 | } |
3044 | 3092 | ||
3045 | if (size != global_trace.entries) | 3093 | if (cpu_id != RING_BUFFER_ALL_CPUS) { |
3046 | ret = __tracing_resize_ring_buffer(size); | 3094 | /* make sure, this cpu is enabled in the mask */ |
3095 | if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) { | ||
3096 | ret = -EINVAL; | ||
3097 | goto out; | ||
3098 | } | ||
3099 | } | ||
3047 | 3100 | ||
3101 | ret = __tracing_resize_ring_buffer(size, cpu_id); | ||
3048 | if (ret < 0) | 3102 | if (ret < 0) |
3049 | ret = -ENOMEM; | 3103 | ret = -ENOMEM; |
3050 | 3104 | ||
3105 | out: | ||
3051 | for_each_tracing_cpu(cpu) { | 3106 | for_each_tracing_cpu(cpu) { |
3052 | if (global_trace.data[cpu]) | 3107 | if (global_trace.data[cpu]) |
3053 | atomic_dec(&global_trace.data[cpu]->disabled); | 3108 | atomic_dec(&global_trace.data[cpu]->disabled); |
@@ -3078,7 +3133,8 @@ int tracing_update_buffers(void) | |||
3078 | 3133 | ||
3079 | mutex_lock(&trace_types_lock); | 3134 | mutex_lock(&trace_types_lock); |
3080 | if (!ring_buffer_expanded) | 3135 | if (!ring_buffer_expanded) |
3081 | ret = __tracing_resize_ring_buffer(trace_buf_size); | 3136 | ret = __tracing_resize_ring_buffer(trace_buf_size, |
3137 | RING_BUFFER_ALL_CPUS); | ||
3082 | mutex_unlock(&trace_types_lock); | 3138 | mutex_unlock(&trace_types_lock); |
3083 | 3139 | ||
3084 | return ret; | 3140 | return ret; |
@@ -3102,7 +3158,8 @@ static int tracing_set_tracer(const char *buf) | |||
3102 | mutex_lock(&trace_types_lock); | 3158 | mutex_lock(&trace_types_lock); |
3103 | 3159 | ||
3104 | if (!ring_buffer_expanded) { | 3160 | if (!ring_buffer_expanded) { |
3105 | ret = __tracing_resize_ring_buffer(trace_buf_size); | 3161 | ret = __tracing_resize_ring_buffer(trace_buf_size, |
3162 | RING_BUFFER_ALL_CPUS); | ||
3106 | if (ret < 0) | 3163 | if (ret < 0) |
3107 | goto out; | 3164 | goto out; |
3108 | ret = 0; | 3165 | ret = 0; |
@@ -3128,8 +3185,8 @@ static int tracing_set_tracer(const char *buf) | |||
3128 | * The max_tr ring buffer has some state (e.g. ring->clock) and | 3185 | * The max_tr ring buffer has some state (e.g. ring->clock) and |
3129 | * we want preserve it. | 3186 | * we want preserve it. |
3130 | */ | 3187 | */ |
3131 | ring_buffer_resize(max_tr.buffer, 1); | 3188 | ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS); |
3132 | max_tr.entries = 1; | 3189 | set_buffer_entries(&max_tr, 1); |
3133 | } | 3190 | } |
3134 | destroy_trace_option_files(topts); | 3191 | destroy_trace_option_files(topts); |
3135 | 3192 | ||
@@ -3137,10 +3194,17 @@ static int tracing_set_tracer(const char *buf) | |||
3137 | 3194 | ||
3138 | topts = create_trace_option_files(current_trace); | 3195 | topts = create_trace_option_files(current_trace); |
3139 | if (current_trace->use_max_tr) { | 3196 | if (current_trace->use_max_tr) { |
3140 | ret = ring_buffer_resize(max_tr.buffer, global_trace.entries); | 3197 | int cpu; |
3141 | if (ret < 0) | 3198 | /* we need to make per cpu buffer sizes equivalent */ |
3142 | goto out; | 3199 | for_each_tracing_cpu(cpu) { |
3143 | max_tr.entries = global_trace.entries; | 3200 | ret = ring_buffer_resize(max_tr.buffer, |
3201 | global_trace.data[cpu]->entries, | ||
3202 | cpu); | ||
3203 | if (ret < 0) | ||
3204 | goto out; | ||
3205 | max_tr.data[cpu]->entries = | ||
3206 | global_trace.data[cpu]->entries; | ||
3207 | } | ||
3144 | } | 3208 | } |
3145 | 3209 | ||
3146 | if (t->init) { | 3210 | if (t->init) { |
@@ -3642,30 +3706,82 @@ out_err: | |||
3642 | goto out; | 3706 | goto out; |
3643 | } | 3707 | } |
3644 | 3708 | ||
3709 | struct ftrace_entries_info { | ||
3710 | struct trace_array *tr; | ||
3711 | int cpu; | ||
3712 | }; | ||
3713 | |||
3714 | static int tracing_entries_open(struct inode *inode, struct file *filp) | ||
3715 | { | ||
3716 | struct ftrace_entries_info *info; | ||
3717 | |||
3718 | if (tracing_disabled) | ||
3719 | return -ENODEV; | ||
3720 | |||
3721 | info = kzalloc(sizeof(*info), GFP_KERNEL); | ||
3722 | if (!info) | ||
3723 | return -ENOMEM; | ||
3724 | |||
3725 | info->tr = &global_trace; | ||
3726 | info->cpu = (unsigned long)inode->i_private; | ||
3727 | |||
3728 | filp->private_data = info; | ||
3729 | |||
3730 | return 0; | ||
3731 | } | ||
3732 | |||
3645 | static ssize_t | 3733 | static ssize_t |
3646 | tracing_entries_read(struct file *filp, char __user *ubuf, | 3734 | tracing_entries_read(struct file *filp, char __user *ubuf, |
3647 | size_t cnt, loff_t *ppos) | 3735 | size_t cnt, loff_t *ppos) |
3648 | { | 3736 | { |
3649 | struct trace_array *tr = filp->private_data; | 3737 | struct ftrace_entries_info *info = filp->private_data; |
3650 | char buf[96]; | 3738 | struct trace_array *tr = info->tr; |
3651 | int r; | 3739 | char buf[64]; |
3740 | int r = 0; | ||
3741 | ssize_t ret; | ||
3652 | 3742 | ||
3653 | mutex_lock(&trace_types_lock); | 3743 | mutex_lock(&trace_types_lock); |
3654 | if (!ring_buffer_expanded) | 3744 | |
3655 | r = sprintf(buf, "%lu (expanded: %lu)\n", | 3745 | if (info->cpu == RING_BUFFER_ALL_CPUS) { |
3656 | tr->entries >> 10, | 3746 | int cpu, buf_size_same; |
3657 | trace_buf_size >> 10); | 3747 | unsigned long size; |
3658 | else | 3748 | |
3659 | r = sprintf(buf, "%lu\n", tr->entries >> 10); | 3749 | size = 0; |
3750 | buf_size_same = 1; | ||
3751 | /* check if all cpu sizes are same */ | ||
3752 | for_each_tracing_cpu(cpu) { | ||
3753 | /* fill in the size from first enabled cpu */ | ||
3754 | if (size == 0) | ||
3755 | size = tr->data[cpu]->entries; | ||
3756 | if (size != tr->data[cpu]->entries) { | ||
3757 | buf_size_same = 0; | ||
3758 | break; | ||
3759 | } | ||
3760 | } | ||
3761 | |||
3762 | if (buf_size_same) { | ||
3763 | if (!ring_buffer_expanded) | ||
3764 | r = sprintf(buf, "%lu (expanded: %lu)\n", | ||
3765 | size >> 10, | ||
3766 | trace_buf_size >> 10); | ||
3767 | else | ||
3768 | r = sprintf(buf, "%lu\n", size >> 10); | ||
3769 | } else | ||
3770 | r = sprintf(buf, "X\n"); | ||
3771 | } else | ||
3772 | r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10); | ||
3773 | |||
3660 | mutex_unlock(&trace_types_lock); | 3774 | mutex_unlock(&trace_types_lock); |
3661 | 3775 | ||
3662 | return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); | 3776 | ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); |
3777 | return ret; | ||
3663 | } | 3778 | } |
3664 | 3779 | ||
3665 | static ssize_t | 3780 | static ssize_t |
3666 | tracing_entries_write(struct file *filp, const char __user *ubuf, | 3781 | tracing_entries_write(struct file *filp, const char __user *ubuf, |
3667 | size_t cnt, loff_t *ppos) | 3782 | size_t cnt, loff_t *ppos) |
3668 | { | 3783 | { |
3784 | struct ftrace_entries_info *info = filp->private_data; | ||
3669 | unsigned long val; | 3785 | unsigned long val; |
3670 | int ret; | 3786 | int ret; |
3671 | 3787 | ||
@@ -3680,7 +3796,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, | |||
3680 | /* value is in KB */ | 3796 | /* value is in KB */ |
3681 | val <<= 10; | 3797 | val <<= 10; |
3682 | 3798 | ||
3683 | ret = tracing_resize_ring_buffer(val); | 3799 | ret = tracing_resize_ring_buffer(val, info->cpu); |
3684 | if (ret < 0) | 3800 | if (ret < 0) |
3685 | return ret; | 3801 | return ret; |
3686 | 3802 | ||
@@ -3689,6 +3805,16 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, | |||
3689 | return cnt; | 3805 | return cnt; |
3690 | } | 3806 | } |
3691 | 3807 | ||
3808 | static int | ||
3809 | tracing_entries_release(struct inode *inode, struct file *filp) | ||
3810 | { | ||
3811 | struct ftrace_entries_info *info = filp->private_data; | ||
3812 | |||
3813 | kfree(info); | ||
3814 | |||
3815 | return 0; | ||
3816 | } | ||
3817 | |||
3692 | static ssize_t | 3818 | static ssize_t |
3693 | tracing_total_entries_read(struct file *filp, char __user *ubuf, | 3819 | tracing_total_entries_read(struct file *filp, char __user *ubuf, |
3694 | size_t cnt, loff_t *ppos) | 3820 | size_t cnt, loff_t *ppos) |
@@ -3700,7 +3826,7 @@ tracing_total_entries_read(struct file *filp, char __user *ubuf, | |||
3700 | 3826 | ||
3701 | mutex_lock(&trace_types_lock); | 3827 | mutex_lock(&trace_types_lock); |
3702 | for_each_tracing_cpu(cpu) { | 3828 | for_each_tracing_cpu(cpu) { |
3703 | size += tr->entries >> 10; | 3829 | size += tr->data[cpu]->entries >> 10; |
3704 | if (!ring_buffer_expanded) | 3830 | if (!ring_buffer_expanded) |
3705 | expanded_size += trace_buf_size >> 10; | 3831 | expanded_size += trace_buf_size >> 10; |
3706 | } | 3832 | } |
@@ -3734,7 +3860,7 @@ tracing_free_buffer_release(struct inode *inode, struct file *filp) | |||
3734 | if (trace_flags & TRACE_ITER_STOP_ON_FREE) | 3860 | if (trace_flags & TRACE_ITER_STOP_ON_FREE) |
3735 | tracing_off(); | 3861 | tracing_off(); |
3736 | /* resize the ring buffer to 0 */ | 3862 | /* resize the ring buffer to 0 */ |
3737 | tracing_resize_ring_buffer(0); | 3863 | tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS); |
3738 | 3864 | ||
3739 | return 0; | 3865 | return 0; |
3740 | } | 3866 | } |
@@ -3933,9 +4059,10 @@ static const struct file_operations tracing_pipe_fops = { | |||
3933 | }; | 4059 | }; |
3934 | 4060 | ||
3935 | static const struct file_operations tracing_entries_fops = { | 4061 | static const struct file_operations tracing_entries_fops = { |
3936 | .open = tracing_open_generic, | 4062 | .open = tracing_entries_open, |
3937 | .read = tracing_entries_read, | 4063 | .read = tracing_entries_read, |
3938 | .write = tracing_entries_write, | 4064 | .write = tracing_entries_write, |
4065 | .release = tracing_entries_release, | ||
3939 | .llseek = generic_file_llseek, | 4066 | .llseek = generic_file_llseek, |
3940 | }; | 4067 | }; |
3941 | 4068 | ||
@@ -4387,6 +4514,9 @@ static void tracing_init_debugfs_percpu(long cpu) | |||
4387 | 4514 | ||
4388 | trace_create_file("stats", 0444, d_cpu, | 4515 | trace_create_file("stats", 0444, d_cpu, |
4389 | (void *) cpu, &tracing_stats_fops); | 4516 | (void *) cpu, &tracing_stats_fops); |
4517 | |||
4518 | trace_create_file("buffer_size_kb", 0444, d_cpu, | ||
4519 | (void *) cpu, &tracing_entries_fops); | ||
4390 | } | 4520 | } |
4391 | 4521 | ||
4392 | #ifdef CONFIG_FTRACE_SELFTEST | 4522 | #ifdef CONFIG_FTRACE_SELFTEST |
@@ -4718,7 +4848,7 @@ static __init int tracer_init_debugfs(void) | |||
4718 | (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops); | 4848 | (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops); |
4719 | 4849 | ||
4720 | trace_create_file("buffer_size_kb", 0644, d_tracer, | 4850 | trace_create_file("buffer_size_kb", 0644, d_tracer, |
4721 | &global_trace, &tracing_entries_fops); | 4851 | (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops); |
4722 | 4852 | ||
4723 | trace_create_file("buffer_total_size_kb", 0444, d_tracer, | 4853 | trace_create_file("buffer_total_size_kb", 0444, d_tracer, |
4724 | &global_trace, &tracing_total_entries_fops); | 4854 | &global_trace, &tracing_total_entries_fops); |
@@ -4957,6 +5087,10 @@ __init static int tracer_alloc_buffers(void) | |||
4957 | if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) | 5087 | if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) |
4958 | goto out_free_buffer_mask; | 5088 | goto out_free_buffer_mask; |
4959 | 5089 | ||
5090 | /* Only allocate trace_printk buffers if a trace_printk exists */ | ||
5091 | if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt) | ||
5092 | trace_printk_init_buffers(); | ||
5093 | |||
4960 | /* To save memory, keep the ring buffer size to its minimum */ | 5094 | /* To save memory, keep the ring buffer size to its minimum */ |
4961 | if (ring_buffer_expanded) | 5095 | if (ring_buffer_expanded) |
4962 | ring_buf_size = trace_buf_size; | 5096 | ring_buf_size = trace_buf_size; |
@@ -4975,7 +5109,6 @@ __init static int tracer_alloc_buffers(void) | |||
4975 | WARN_ON(1); | 5109 | WARN_ON(1); |
4976 | goto out_free_cpumask; | 5110 | goto out_free_cpumask; |
4977 | } | 5111 | } |
4978 | global_trace.entries = ring_buffer_size(global_trace.buffer); | ||
4979 | if (global_trace.buffer_disabled) | 5112 | if (global_trace.buffer_disabled) |
4980 | tracing_off(); | 5113 | tracing_off(); |
4981 | 5114 | ||
@@ -4988,7 +5121,6 @@ __init static int tracer_alloc_buffers(void) | |||
4988 | ring_buffer_free(global_trace.buffer); | 5121 | ring_buffer_free(global_trace.buffer); |
4989 | goto out_free_cpumask; | 5122 | goto out_free_cpumask; |
4990 | } | 5123 | } |
4991 | max_tr.entries = 1; | ||
4992 | #endif | 5124 | #endif |
4993 | 5125 | ||
4994 | /* Allocate the first page for all buffers */ | 5126 | /* Allocate the first page for all buffers */ |
@@ -4997,6 +5129,11 @@ __init static int tracer_alloc_buffers(void) | |||
4997 | max_tr.data[i] = &per_cpu(max_tr_data, i); | 5129 | max_tr.data[i] = &per_cpu(max_tr_data, i); |
4998 | } | 5130 | } |
4999 | 5131 | ||
5132 | set_buffer_entries(&global_trace, ring_buf_size); | ||
5133 | #ifdef CONFIG_TRACER_MAX_TRACE | ||
5134 | set_buffer_entries(&max_tr, 1); | ||
5135 | #endif | ||
5136 | |||
5000 | trace_init_cmdlines(); | 5137 | trace_init_cmdlines(); |
5001 | 5138 | ||
5002 | register_tracer(&nop_trace); | 5139 | register_tracer(&nop_trace); |
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index f95d65da6db..6c6f7933eed 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
@@ -131,6 +131,7 @@ struct trace_array_cpu { | |||
131 | atomic_t disabled; | 131 | atomic_t disabled; |
132 | void *buffer_page; /* ring buffer spare */ | 132 | void *buffer_page; /* ring buffer spare */ |
133 | 133 | ||
134 | unsigned long entries; | ||
134 | unsigned long saved_latency; | 135 | unsigned long saved_latency; |
135 | unsigned long critical_start; | 136 | unsigned long critical_start; |
136 | unsigned long critical_end; | 137 | unsigned long critical_end; |
@@ -152,7 +153,6 @@ struct trace_array_cpu { | |||
152 | */ | 153 | */ |
153 | struct trace_array { | 154 | struct trace_array { |
154 | struct ring_buffer *buffer; | 155 | struct ring_buffer *buffer; |
155 | unsigned long entries; | ||
156 | int cpu; | 156 | int cpu; |
157 | int buffer_disabled; | 157 | int buffer_disabled; |
158 | cycle_t time_start; | 158 | cycle_t time_start; |
@@ -826,6 +826,8 @@ extern struct list_head ftrace_events; | |||
826 | extern const char *__start___trace_bprintk_fmt[]; | 826 | extern const char *__start___trace_bprintk_fmt[]; |
827 | extern const char *__stop___trace_bprintk_fmt[]; | 827 | extern const char *__stop___trace_bprintk_fmt[]; |
828 | 828 | ||
829 | void trace_printk_init_buffers(void); | ||
830 | |||
829 | #undef FTRACE_ENTRY | 831 | #undef FTRACE_ENTRY |
830 | #define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \ | 832 | #define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \ |
831 | extern struct ftrace_event_call \ | 833 | extern struct ftrace_event_call \ |
diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c index 6fd4ffd042f..a9077c1b4ad 100644 --- a/kernel/trace/trace_printk.c +++ b/kernel/trace/trace_printk.c | |||
@@ -51,6 +51,10 @@ void hold_module_trace_bprintk_format(const char **start, const char **end) | |||
51 | const char **iter; | 51 | const char **iter; |
52 | char *fmt; | 52 | char *fmt; |
53 | 53 | ||
54 | /* allocate the trace_printk per cpu buffers */ | ||
55 | if (start != end) | ||
56 | trace_printk_init_buffers(); | ||
57 | |||
54 | mutex_lock(&btrace_mutex); | 58 | mutex_lock(&btrace_mutex); |
55 | for (iter = start; iter < end; iter++) { | 59 | for (iter = start; iter < end; iter++) { |
56 | struct trace_bprintk_fmt *tb_fmt = lookup_format(*iter); | 60 | struct trace_bprintk_fmt *tb_fmt = lookup_format(*iter); |