diff options
Diffstat (limited to 'kernel/trace')
| -rw-r--r-- | kernel/trace/ftrace.c | 123 | ||||
| -rw-r--r-- | kernel/trace/ring_buffer.c | 21 | ||||
| -rw-r--r-- | kernel/trace/trace_event_perf.c | 28 | ||||
| -rw-r--r-- | kernel/trace/trace_events.c | 55 | ||||
| -rw-r--r-- | kernel/trace/trace_functions_graph.c | 126 | ||||
| -rw-r--r-- | kernel/trace/trace_workqueue.c | 10 |
6 files changed, 257 insertions, 106 deletions
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index fa7ece649fe1..65fb077ea79c 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
| @@ -884,10 +884,8 @@ enum { | |||
| 884 | FTRACE_ENABLE_CALLS = (1 << 0), | 884 | FTRACE_ENABLE_CALLS = (1 << 0), |
| 885 | FTRACE_DISABLE_CALLS = (1 << 1), | 885 | FTRACE_DISABLE_CALLS = (1 << 1), |
| 886 | FTRACE_UPDATE_TRACE_FUNC = (1 << 2), | 886 | FTRACE_UPDATE_TRACE_FUNC = (1 << 2), |
| 887 | FTRACE_ENABLE_MCOUNT = (1 << 3), | 887 | FTRACE_START_FUNC_RET = (1 << 3), |
| 888 | FTRACE_DISABLE_MCOUNT = (1 << 4), | 888 | FTRACE_STOP_FUNC_RET = (1 << 4), |
| 889 | FTRACE_START_FUNC_RET = (1 << 5), | ||
| 890 | FTRACE_STOP_FUNC_RET = (1 << 6), | ||
| 891 | }; | 889 | }; |
| 892 | 890 | ||
| 893 | static int ftrace_filtered; | 891 | static int ftrace_filtered; |
| @@ -1226,8 +1224,6 @@ static void ftrace_shutdown(int command) | |||
| 1226 | 1224 | ||
| 1227 | static void ftrace_startup_sysctl(void) | 1225 | static void ftrace_startup_sysctl(void) |
| 1228 | { | 1226 | { |
| 1229 | int command = FTRACE_ENABLE_MCOUNT; | ||
| 1230 | |||
| 1231 | if (unlikely(ftrace_disabled)) | 1227 | if (unlikely(ftrace_disabled)) |
| 1232 | return; | 1228 | return; |
| 1233 | 1229 | ||
| @@ -1235,23 +1231,17 @@ static void ftrace_startup_sysctl(void) | |||
| 1235 | saved_ftrace_func = NULL; | 1231 | saved_ftrace_func = NULL; |
| 1236 | /* ftrace_start_up is true if we want ftrace running */ | 1232 | /* ftrace_start_up is true if we want ftrace running */ |
| 1237 | if (ftrace_start_up) | 1233 | if (ftrace_start_up) |
| 1238 | command |= FTRACE_ENABLE_CALLS; | 1234 | ftrace_run_update_code(FTRACE_ENABLE_CALLS); |
| 1239 | |||
| 1240 | ftrace_run_update_code(command); | ||
| 1241 | } | 1235 | } |
| 1242 | 1236 | ||
| 1243 | static void ftrace_shutdown_sysctl(void) | 1237 | static void ftrace_shutdown_sysctl(void) |
| 1244 | { | 1238 | { |
| 1245 | int command = FTRACE_DISABLE_MCOUNT; | ||
| 1246 | |||
| 1247 | if (unlikely(ftrace_disabled)) | 1239 | if (unlikely(ftrace_disabled)) |
| 1248 | return; | 1240 | return; |
| 1249 | 1241 | ||
| 1250 | /* ftrace_start_up is true if ftrace is running */ | 1242 | /* ftrace_start_up is true if ftrace is running */ |
| 1251 | if (ftrace_start_up) | 1243 | if (ftrace_start_up) |
| 1252 | command |= FTRACE_DISABLE_CALLS; | 1244 | ftrace_run_update_code(FTRACE_DISABLE_CALLS); |
| 1253 | |||
| 1254 | ftrace_run_update_code(command); | ||
| 1255 | } | 1245 | } |
| 1256 | 1246 | ||
| 1257 | static cycle_t ftrace_update_time; | 1247 | static cycle_t ftrace_update_time; |
| @@ -1368,24 +1358,29 @@ enum { | |||
| 1368 | #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ | 1358 | #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ |
| 1369 | 1359 | ||
| 1370 | struct ftrace_iterator { | 1360 | struct ftrace_iterator { |
| 1371 | struct ftrace_page *pg; | 1361 | loff_t pos; |
| 1372 | int hidx; | 1362 | loff_t func_pos; |
| 1373 | int idx; | 1363 | struct ftrace_page *pg; |
| 1374 | unsigned flags; | 1364 | struct dyn_ftrace *func; |
| 1375 | struct trace_parser parser; | 1365 | struct ftrace_func_probe *probe; |
| 1366 | struct trace_parser parser; | ||
| 1367 | int hidx; | ||
| 1368 | int idx; | ||
| 1369 | unsigned flags; | ||
| 1376 | }; | 1370 | }; |
| 1377 | 1371 | ||
| 1378 | static void * | 1372 | static void * |
| 1379 | t_hash_next(struct seq_file *m, void *v, loff_t *pos) | 1373 | t_hash_next(struct seq_file *m, loff_t *pos) |
| 1380 | { | 1374 | { |
| 1381 | struct ftrace_iterator *iter = m->private; | 1375 | struct ftrace_iterator *iter = m->private; |
| 1382 | struct hlist_node *hnd = v; | 1376 | struct hlist_node *hnd = NULL; |
| 1383 | struct hlist_head *hhd; | 1377 | struct hlist_head *hhd; |
| 1384 | 1378 | ||
| 1385 | WARN_ON(!(iter->flags & FTRACE_ITER_HASH)); | ||
| 1386 | |||
| 1387 | (*pos)++; | 1379 | (*pos)++; |
| 1380 | iter->pos = *pos; | ||
| 1388 | 1381 | ||
| 1382 | if (iter->probe) | ||
| 1383 | hnd = &iter->probe->node; | ||
| 1389 | retry: | 1384 | retry: |
| 1390 | if (iter->hidx >= FTRACE_FUNC_HASHSIZE) | 1385 | if (iter->hidx >= FTRACE_FUNC_HASHSIZE) |
| 1391 | return NULL; | 1386 | return NULL; |
| @@ -1408,7 +1403,12 @@ t_hash_next(struct seq_file *m, void *v, loff_t *pos) | |||
| 1408 | } | 1403 | } |
| 1409 | } | 1404 | } |
| 1410 | 1405 | ||
| 1411 | return hnd; | 1406 | if (WARN_ON_ONCE(!hnd)) |
| 1407 | return NULL; | ||
| 1408 | |||
| 1409 | iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node); | ||
| 1410 | |||
| 1411 | return iter; | ||
| 1412 | } | 1412 | } |
| 1413 | 1413 | ||
| 1414 | static void *t_hash_start(struct seq_file *m, loff_t *pos) | 1414 | static void *t_hash_start(struct seq_file *m, loff_t *pos) |
| @@ -1417,26 +1417,32 @@ static void *t_hash_start(struct seq_file *m, loff_t *pos) | |||
| 1417 | void *p = NULL; | 1417 | void *p = NULL; |
| 1418 | loff_t l; | 1418 | loff_t l; |
| 1419 | 1419 | ||
| 1420 | if (!(iter->flags & FTRACE_ITER_HASH)) | 1420 | if (iter->func_pos > *pos) |
| 1421 | *pos = 0; | 1421 | return NULL; |
| 1422 | |||
| 1423 | iter->flags |= FTRACE_ITER_HASH; | ||
| 1424 | 1422 | ||
| 1425 | iter->hidx = 0; | 1423 | iter->hidx = 0; |
| 1426 | for (l = 0; l <= *pos; ) { | 1424 | for (l = 0; l <= (*pos - iter->func_pos); ) { |
| 1427 | p = t_hash_next(m, p, &l); | 1425 | p = t_hash_next(m, &l); |
| 1428 | if (!p) | 1426 | if (!p) |
| 1429 | break; | 1427 | break; |
| 1430 | } | 1428 | } |
| 1431 | return p; | 1429 | if (!p) |
| 1430 | return NULL; | ||
| 1431 | |||
| 1432 | /* Only set this if we have an item */ | ||
| 1433 | iter->flags |= FTRACE_ITER_HASH; | ||
| 1434 | |||
| 1435 | return iter; | ||
| 1432 | } | 1436 | } |
| 1433 | 1437 | ||
| 1434 | static int t_hash_show(struct seq_file *m, void *v) | 1438 | static int |
| 1439 | t_hash_show(struct seq_file *m, struct ftrace_iterator *iter) | ||
| 1435 | { | 1440 | { |
| 1436 | struct ftrace_func_probe *rec; | 1441 | struct ftrace_func_probe *rec; |
| 1437 | struct hlist_node *hnd = v; | ||
| 1438 | 1442 | ||
| 1439 | rec = hlist_entry(hnd, struct ftrace_func_probe, node); | 1443 | rec = iter->probe; |
| 1444 | if (WARN_ON_ONCE(!rec)) | ||
| 1445 | return -EIO; | ||
| 1440 | 1446 | ||
| 1441 | if (rec->ops->print) | 1447 | if (rec->ops->print) |
| 1442 | return rec->ops->print(m, rec->ip, rec->ops, rec->data); | 1448 | return rec->ops->print(m, rec->ip, rec->ops, rec->data); |
| @@ -1457,12 +1463,13 @@ t_next(struct seq_file *m, void *v, loff_t *pos) | |||
| 1457 | struct dyn_ftrace *rec = NULL; | 1463 | struct dyn_ftrace *rec = NULL; |
| 1458 | 1464 | ||
| 1459 | if (iter->flags & FTRACE_ITER_HASH) | 1465 | if (iter->flags & FTRACE_ITER_HASH) |
| 1460 | return t_hash_next(m, v, pos); | 1466 | return t_hash_next(m, pos); |
| 1461 | 1467 | ||
| 1462 | (*pos)++; | 1468 | (*pos)++; |
| 1469 | iter->pos = *pos; | ||
| 1463 | 1470 | ||
| 1464 | if (iter->flags & FTRACE_ITER_PRINTALL) | 1471 | if (iter->flags & FTRACE_ITER_PRINTALL) |
| 1465 | return NULL; | 1472 | return t_hash_start(m, pos); |
| 1466 | 1473 | ||
| 1467 | retry: | 1474 | retry: |
| 1468 | if (iter->idx >= iter->pg->index) { | 1475 | if (iter->idx >= iter->pg->index) { |
| @@ -1491,7 +1498,20 @@ t_next(struct seq_file *m, void *v, loff_t *pos) | |||
| 1491 | } | 1498 | } |
| 1492 | } | 1499 | } |
| 1493 | 1500 | ||
| 1494 | return rec; | 1501 | if (!rec) |
| 1502 | return t_hash_start(m, pos); | ||
| 1503 | |||
| 1504 | iter->func_pos = *pos; | ||
| 1505 | iter->func = rec; | ||
| 1506 | |||
| 1507 | return iter; | ||
| 1508 | } | ||
| 1509 | |||
| 1510 | static void reset_iter_read(struct ftrace_iterator *iter) | ||
| 1511 | { | ||
| 1512 | iter->pos = 0; | ||
| 1513 | iter->func_pos = 0; | ||
| 1514 | iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH); | ||
| 1495 | } | 1515 | } |
| 1496 | 1516 | ||
| 1497 | static void *t_start(struct seq_file *m, loff_t *pos) | 1517 | static void *t_start(struct seq_file *m, loff_t *pos) |
| @@ -1502,6 +1522,12 @@ static void *t_start(struct seq_file *m, loff_t *pos) | |||
| 1502 | 1522 | ||
| 1503 | mutex_lock(&ftrace_lock); | 1523 | mutex_lock(&ftrace_lock); |
| 1504 | /* | 1524 | /* |
| 1525 | * If an lseek was done, then reset and start from beginning. | ||
| 1526 | */ | ||
| 1527 | if (*pos < iter->pos) | ||
| 1528 | reset_iter_read(iter); | ||
| 1529 | |||
| 1530 | /* | ||
| 1505 | * For set_ftrace_filter reading, if we have the filter | 1531 | * For set_ftrace_filter reading, if we have the filter |
| 1506 | * off, we can short cut and just print out that all | 1532 | * off, we can short cut and just print out that all |
| 1507 | * functions are enabled. | 1533 | * functions are enabled. |
| @@ -1518,6 +1544,11 @@ static void *t_start(struct seq_file *m, loff_t *pos) | |||
| 1518 | if (iter->flags & FTRACE_ITER_HASH) | 1544 | if (iter->flags & FTRACE_ITER_HASH) |
| 1519 | return t_hash_start(m, pos); | 1545 | return t_hash_start(m, pos); |
| 1520 | 1546 | ||
| 1547 | /* | ||
| 1548 | * Unfortunately, we need to restart at ftrace_pages_start | ||
| 1549 | * every time we let go of the ftrace_mutex. This is because | ||
| 1550 | * those pointers can change without the lock. | ||
| 1551 | */ | ||
| 1521 | iter->pg = ftrace_pages_start; | 1552 | iter->pg = ftrace_pages_start; |
| 1522 | iter->idx = 0; | 1553 | iter->idx = 0; |
| 1523 | for (l = 0; l <= *pos; ) { | 1554 | for (l = 0; l <= *pos; ) { |
| @@ -1526,10 +1557,14 @@ static void *t_start(struct seq_file *m, loff_t *pos) | |||
| 1526 | break; | 1557 | break; |
| 1527 | } | 1558 | } |
| 1528 | 1559 | ||
| 1529 | if (!p && iter->flags & FTRACE_ITER_FILTER) | 1560 | if (!p) { |
| 1530 | return t_hash_start(m, pos); | 1561 | if (iter->flags & FTRACE_ITER_FILTER) |
| 1562 | return t_hash_start(m, pos); | ||
| 1531 | 1563 | ||
| 1532 | return p; | 1564 | return NULL; |
| 1565 | } | ||
| 1566 | |||
| 1567 | return iter; | ||
| 1533 | } | 1568 | } |
| 1534 | 1569 | ||
| 1535 | static void t_stop(struct seq_file *m, void *p) | 1570 | static void t_stop(struct seq_file *m, void *p) |
| @@ -1540,16 +1575,18 @@ static void t_stop(struct seq_file *m, void *p) | |||
| 1540 | static int t_show(struct seq_file *m, void *v) | 1575 | static int t_show(struct seq_file *m, void *v) |
| 1541 | { | 1576 | { |
| 1542 | struct ftrace_iterator *iter = m->private; | 1577 | struct ftrace_iterator *iter = m->private; |
| 1543 | struct dyn_ftrace *rec = v; | 1578 | struct dyn_ftrace *rec; |
| 1544 | 1579 | ||
| 1545 | if (iter->flags & FTRACE_ITER_HASH) | 1580 | if (iter->flags & FTRACE_ITER_HASH) |
| 1546 | return t_hash_show(m, v); | 1581 | return t_hash_show(m, iter); |
| 1547 | 1582 | ||
| 1548 | if (iter->flags & FTRACE_ITER_PRINTALL) { | 1583 | if (iter->flags & FTRACE_ITER_PRINTALL) { |
| 1549 | seq_printf(m, "#### all functions enabled ####\n"); | 1584 | seq_printf(m, "#### all functions enabled ####\n"); |
| 1550 | return 0; | 1585 | return 0; |
| 1551 | } | 1586 | } |
| 1552 | 1587 | ||
| 1588 | rec = iter->func; | ||
| 1589 | |||
| 1553 | if (!rec) | 1590 | if (!rec) |
| 1554 | return 0; | 1591 | return 0; |
| 1555 | 1592 | ||
| @@ -2418,7 +2455,7 @@ static const struct file_operations ftrace_filter_fops = { | |||
| 2418 | .open = ftrace_filter_open, | 2455 | .open = ftrace_filter_open, |
| 2419 | .read = seq_read, | 2456 | .read = seq_read, |
| 2420 | .write = ftrace_filter_write, | 2457 | .write = ftrace_filter_write, |
| 2421 | .llseek = no_llseek, | 2458 | .llseek = ftrace_regex_lseek, |
| 2422 | .release = ftrace_filter_release, | 2459 | .release = ftrace_filter_release, |
| 2423 | }; | 2460 | }; |
| 2424 | 2461 | ||
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 492197e2f86c..4e2f03410377 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
| @@ -2606,6 +2606,19 @@ void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu) | |||
| 2606 | } | 2606 | } |
| 2607 | EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu); | 2607 | EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu); |
| 2608 | 2608 | ||
| 2609 | /* | ||
| 2610 | * The total entries in the ring buffer is the running counter | ||
| 2611 | * of entries entered into the ring buffer, minus the sum of | ||
| 2612 | * the entries read from the ring buffer and the number of | ||
| 2613 | * entries that were overwritten. | ||
| 2614 | */ | ||
| 2615 | static inline unsigned long | ||
| 2616 | rb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer) | ||
| 2617 | { | ||
| 2618 | return local_read(&cpu_buffer->entries) - | ||
| 2619 | (local_read(&cpu_buffer->overrun) + cpu_buffer->read); | ||
| 2620 | } | ||
| 2621 | |||
| 2609 | /** | 2622 | /** |
| 2610 | * ring_buffer_entries_cpu - get the number of entries in a cpu buffer | 2623 | * ring_buffer_entries_cpu - get the number of entries in a cpu buffer |
| 2611 | * @buffer: The ring buffer | 2624 | * @buffer: The ring buffer |
| @@ -2614,16 +2627,13 @@ EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu); | |||
| 2614 | unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu) | 2627 | unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu) |
| 2615 | { | 2628 | { |
| 2616 | struct ring_buffer_per_cpu *cpu_buffer; | 2629 | struct ring_buffer_per_cpu *cpu_buffer; |
| 2617 | unsigned long ret; | ||
| 2618 | 2630 | ||
| 2619 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | 2631 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) |
| 2620 | return 0; | 2632 | return 0; |
| 2621 | 2633 | ||
| 2622 | cpu_buffer = buffer->buffers[cpu]; | 2634 | cpu_buffer = buffer->buffers[cpu]; |
| 2623 | ret = (local_read(&cpu_buffer->entries) - local_read(&cpu_buffer->overrun)) | ||
| 2624 | - cpu_buffer->read; | ||
| 2625 | 2635 | ||
| 2626 | return ret; | 2636 | return rb_num_of_entries(cpu_buffer); |
| 2627 | } | 2637 | } |
| 2628 | EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu); | 2638 | EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu); |
| 2629 | 2639 | ||
| @@ -2684,8 +2694,7 @@ unsigned long ring_buffer_entries(struct ring_buffer *buffer) | |||
| 2684 | /* if you care about this being correct, lock the buffer */ | 2694 | /* if you care about this being correct, lock the buffer */ |
| 2685 | for_each_buffer_cpu(buffer, cpu) { | 2695 | for_each_buffer_cpu(buffer, cpu) { |
| 2686 | cpu_buffer = buffer->buffers[cpu]; | 2696 | cpu_buffer = buffer->buffers[cpu]; |
| 2687 | entries += (local_read(&cpu_buffer->entries) - | 2697 | entries += rb_num_of_entries(cpu_buffer); |
| 2688 | local_read(&cpu_buffer->overrun)) - cpu_buffer->read; | ||
| 2689 | } | 2698 | } |
| 2690 | 2699 | ||
| 2691 | return entries; | 2700 | return entries; |
diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index 31cc4cb0dbf2..39c059ca670e 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c | |||
| @@ -9,7 +9,7 @@ | |||
| 9 | #include <linux/kprobes.h> | 9 | #include <linux/kprobes.h> |
| 10 | #include "trace.h" | 10 | #include "trace.h" |
| 11 | 11 | ||
| 12 | static char *perf_trace_buf[4]; | 12 | static char __percpu *perf_trace_buf[PERF_NR_CONTEXTS]; |
| 13 | 13 | ||
| 14 | /* | 14 | /* |
| 15 | * Force it to be aligned to unsigned long to avoid misaligned accesses | 15 | * Force it to be aligned to unsigned long to avoid misaligned accesses |
| @@ -24,7 +24,7 @@ static int total_ref_count; | |||
| 24 | static int perf_trace_event_init(struct ftrace_event_call *tp_event, | 24 | static int perf_trace_event_init(struct ftrace_event_call *tp_event, |
| 25 | struct perf_event *p_event) | 25 | struct perf_event *p_event) |
| 26 | { | 26 | { |
| 27 | struct hlist_head *list; | 27 | struct hlist_head __percpu *list; |
| 28 | int ret = -ENOMEM; | 28 | int ret = -ENOMEM; |
| 29 | int cpu; | 29 | int cpu; |
| 30 | 30 | ||
| @@ -42,11 +42,11 @@ static int perf_trace_event_init(struct ftrace_event_call *tp_event, | |||
| 42 | tp_event->perf_events = list; | 42 | tp_event->perf_events = list; |
| 43 | 43 | ||
| 44 | if (!total_ref_count) { | 44 | if (!total_ref_count) { |
| 45 | char *buf; | 45 | char __percpu *buf; |
| 46 | int i; | 46 | int i; |
| 47 | 47 | ||
| 48 | for (i = 0; i < 4; i++) { | 48 | for (i = 0; i < PERF_NR_CONTEXTS; i++) { |
| 49 | buf = (char *)alloc_percpu(perf_trace_t); | 49 | buf = (char __percpu *)alloc_percpu(perf_trace_t); |
| 50 | if (!buf) | 50 | if (!buf) |
| 51 | goto fail; | 51 | goto fail; |
| 52 | 52 | ||
| @@ -65,7 +65,7 @@ fail: | |||
| 65 | if (!total_ref_count) { | 65 | if (!total_ref_count) { |
| 66 | int i; | 66 | int i; |
| 67 | 67 | ||
| 68 | for (i = 0; i < 4; i++) { | 68 | for (i = 0; i < PERF_NR_CONTEXTS; i++) { |
| 69 | free_percpu(perf_trace_buf[i]); | 69 | free_percpu(perf_trace_buf[i]); |
| 70 | perf_trace_buf[i] = NULL; | 70 | perf_trace_buf[i] = NULL; |
| 71 | } | 71 | } |
| @@ -101,22 +101,26 @@ int perf_trace_init(struct perf_event *p_event) | |||
| 101 | return ret; | 101 | return ret; |
| 102 | } | 102 | } |
| 103 | 103 | ||
| 104 | int perf_trace_enable(struct perf_event *p_event) | 104 | int perf_trace_add(struct perf_event *p_event, int flags) |
| 105 | { | 105 | { |
| 106 | struct ftrace_event_call *tp_event = p_event->tp_event; | 106 | struct ftrace_event_call *tp_event = p_event->tp_event; |
| 107 | struct hlist_head __percpu *pcpu_list; | ||
| 107 | struct hlist_head *list; | 108 | struct hlist_head *list; |
| 108 | 109 | ||
| 109 | list = tp_event->perf_events; | 110 | pcpu_list = tp_event->perf_events; |
| 110 | if (WARN_ON_ONCE(!list)) | 111 | if (WARN_ON_ONCE(!pcpu_list)) |
| 111 | return -EINVAL; | 112 | return -EINVAL; |
| 112 | 113 | ||
| 113 | list = this_cpu_ptr(list); | 114 | if (!(flags & PERF_EF_START)) |
| 115 | p_event->hw.state = PERF_HES_STOPPED; | ||
| 116 | |||
| 117 | list = this_cpu_ptr(pcpu_list); | ||
| 114 | hlist_add_head_rcu(&p_event->hlist_entry, list); | 118 | hlist_add_head_rcu(&p_event->hlist_entry, list); |
| 115 | 119 | ||
| 116 | return 0; | 120 | return 0; |
| 117 | } | 121 | } |
| 118 | 122 | ||
| 119 | void perf_trace_disable(struct perf_event *p_event) | 123 | void perf_trace_del(struct perf_event *p_event, int flags) |
| 120 | { | 124 | { |
| 121 | hlist_del_rcu(&p_event->hlist_entry); | 125 | hlist_del_rcu(&p_event->hlist_entry); |
| 122 | } | 126 | } |
| @@ -142,7 +146,7 @@ void perf_trace_destroy(struct perf_event *p_event) | |||
| 142 | tp_event->perf_events = NULL; | 146 | tp_event->perf_events = NULL; |
| 143 | 147 | ||
| 144 | if (!--total_ref_count) { | 148 | if (!--total_ref_count) { |
| 145 | for (i = 0; i < 4; i++) { | 149 | for (i = 0; i < PERF_NR_CONTEXTS; i++) { |
| 146 | free_percpu(perf_trace_buf[i]); | 150 | free_percpu(perf_trace_buf[i]); |
| 147 | perf_trace_buf[i] = NULL; | 151 | perf_trace_buf[i] = NULL; |
| 148 | } | 152 | } |
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 4c758f146328..398c0e8b332c 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c | |||
| @@ -600,21 +600,29 @@ out: | |||
| 600 | 600 | ||
| 601 | enum { | 601 | enum { |
| 602 | FORMAT_HEADER = 1, | 602 | FORMAT_HEADER = 1, |
| 603 | FORMAT_PRINTFMT = 2, | 603 | FORMAT_FIELD_SEPERATOR = 2, |
| 604 | FORMAT_PRINTFMT = 3, | ||
| 604 | }; | 605 | }; |
| 605 | 606 | ||
| 606 | static void *f_next(struct seq_file *m, void *v, loff_t *pos) | 607 | static void *f_next(struct seq_file *m, void *v, loff_t *pos) |
| 607 | { | 608 | { |
| 608 | struct ftrace_event_call *call = m->private; | 609 | struct ftrace_event_call *call = m->private; |
| 609 | struct ftrace_event_field *field; | 610 | struct ftrace_event_field *field; |
| 610 | struct list_head *head; | 611 | struct list_head *common_head = &ftrace_common_fields; |
| 612 | struct list_head *head = trace_get_fields(call); | ||
| 611 | 613 | ||
| 612 | (*pos)++; | 614 | (*pos)++; |
| 613 | 615 | ||
| 614 | switch ((unsigned long)v) { | 616 | switch ((unsigned long)v) { |
| 615 | case FORMAT_HEADER: | 617 | case FORMAT_HEADER: |
| 616 | head = &ftrace_common_fields; | 618 | if (unlikely(list_empty(common_head))) |
| 619 | return NULL; | ||
| 620 | |||
| 621 | field = list_entry(common_head->prev, | ||
| 622 | struct ftrace_event_field, link); | ||
| 623 | return field; | ||
| 617 | 624 | ||
| 625 | case FORMAT_FIELD_SEPERATOR: | ||
| 618 | if (unlikely(list_empty(head))) | 626 | if (unlikely(list_empty(head))) |
| 619 | return NULL; | 627 | return NULL; |
| 620 | 628 | ||
| @@ -626,31 +634,10 @@ static void *f_next(struct seq_file *m, void *v, loff_t *pos) | |||
| 626 | return NULL; | 634 | return NULL; |
| 627 | } | 635 | } |
| 628 | 636 | ||
| 629 | head = trace_get_fields(call); | ||
| 630 | |||
| 631 | /* | ||
| 632 | * To separate common fields from event fields, the | ||
| 633 | * LSB is set on the first event field. Clear it in case. | ||
| 634 | */ | ||
| 635 | v = (void *)((unsigned long)v & ~1L); | ||
| 636 | |||
| 637 | field = v; | 637 | field = v; |
| 638 | /* | 638 | if (field->link.prev == common_head) |
| 639 | * If this is a common field, and at the end of the list, then | 639 | return (void *)FORMAT_FIELD_SEPERATOR; |
| 640 | * continue with main list. | 640 | else if (field->link.prev == head) |
| 641 | */ | ||
| 642 | if (field->link.prev == &ftrace_common_fields) { | ||
| 643 | if (unlikely(list_empty(head))) | ||
| 644 | return NULL; | ||
| 645 | field = list_entry(head->prev, struct ftrace_event_field, link); | ||
| 646 | /* Set the LSB to notify f_show to print an extra newline */ | ||
| 647 | field = (struct ftrace_event_field *) | ||
| 648 | ((unsigned long)field | 1); | ||
| 649 | return field; | ||
| 650 | } | ||
| 651 | |||
| 652 | /* If we are done tell f_show to print the format */ | ||
| 653 | if (field->link.prev == head) | ||
| 654 | return (void *)FORMAT_PRINTFMT; | 641 | return (void *)FORMAT_PRINTFMT; |
| 655 | 642 | ||
| 656 | field = list_entry(field->link.prev, struct ftrace_event_field, link); | 643 | field = list_entry(field->link.prev, struct ftrace_event_field, link); |
| @@ -688,22 +675,16 @@ static int f_show(struct seq_file *m, void *v) | |||
| 688 | seq_printf(m, "format:\n"); | 675 | seq_printf(m, "format:\n"); |
| 689 | return 0; | 676 | return 0; |
| 690 | 677 | ||
| 678 | case FORMAT_FIELD_SEPERATOR: | ||
| 679 | seq_putc(m, '\n'); | ||
| 680 | return 0; | ||
| 681 | |||
| 691 | case FORMAT_PRINTFMT: | 682 | case FORMAT_PRINTFMT: |
| 692 | seq_printf(m, "\nprint fmt: %s\n", | 683 | seq_printf(m, "\nprint fmt: %s\n", |
| 693 | call->print_fmt); | 684 | call->print_fmt); |
| 694 | return 0; | 685 | return 0; |
| 695 | } | 686 | } |
| 696 | 687 | ||
| 697 | /* | ||
| 698 | * To separate common fields from event fields, the | ||
| 699 | * LSB is set on the first event field. Clear it and | ||
| 700 | * print a newline if it is set. | ||
| 701 | */ | ||
| 702 | if ((unsigned long)v & 1) { | ||
| 703 | seq_putc(m, '\n'); | ||
| 704 | v = (void *)((unsigned long)v & ~1L); | ||
| 705 | } | ||
| 706 | |||
| 707 | field = v; | 688 | field = v; |
| 708 | 689 | ||
| 709 | /* | 690 | /* |
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 6f233698518e..02c708ae0d42 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c | |||
| @@ -15,15 +15,19 @@ | |||
| 15 | #include "trace.h" | 15 | #include "trace.h" |
| 16 | #include "trace_output.h" | 16 | #include "trace_output.h" |
| 17 | 17 | ||
| 18 | /* When set, irq functions will be ignored */ | ||
| 19 | static int ftrace_graph_skip_irqs; | ||
| 20 | |||
| 18 | struct fgraph_cpu_data { | 21 | struct fgraph_cpu_data { |
| 19 | pid_t last_pid; | 22 | pid_t last_pid; |
| 20 | int depth; | 23 | int depth; |
| 24 | int depth_irq; | ||
| 21 | int ignore; | 25 | int ignore; |
| 22 | unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH]; | 26 | unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH]; |
| 23 | }; | 27 | }; |
| 24 | 28 | ||
| 25 | struct fgraph_data { | 29 | struct fgraph_data { |
| 26 | struct fgraph_cpu_data *cpu_data; | 30 | struct fgraph_cpu_data __percpu *cpu_data; |
| 27 | 31 | ||
| 28 | /* Place to preserve last processed entry. */ | 32 | /* Place to preserve last processed entry. */ |
| 29 | struct ftrace_graph_ent_entry ent; | 33 | struct ftrace_graph_ent_entry ent; |
| @@ -41,6 +45,7 @@ struct fgraph_data { | |||
| 41 | #define TRACE_GRAPH_PRINT_PROC 0x8 | 45 | #define TRACE_GRAPH_PRINT_PROC 0x8 |
| 42 | #define TRACE_GRAPH_PRINT_DURATION 0x10 | 46 | #define TRACE_GRAPH_PRINT_DURATION 0x10 |
| 43 | #define TRACE_GRAPH_PRINT_ABS_TIME 0x20 | 47 | #define TRACE_GRAPH_PRINT_ABS_TIME 0x20 |
| 48 | #define TRACE_GRAPH_PRINT_IRQS 0x40 | ||
| 44 | 49 | ||
| 45 | static struct tracer_opt trace_opts[] = { | 50 | static struct tracer_opt trace_opts[] = { |
| 46 | /* Display overruns? (for self-debug purpose) */ | 51 | /* Display overruns? (for self-debug purpose) */ |
| @@ -55,13 +60,15 @@ static struct tracer_opt trace_opts[] = { | |||
| 55 | { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) }, | 60 | { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) }, |
| 56 | /* Display absolute time of an entry */ | 61 | /* Display absolute time of an entry */ |
| 57 | { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) }, | 62 | { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) }, |
| 63 | /* Display interrupts */ | ||
| 64 | { TRACER_OPT(funcgraph-irqs, TRACE_GRAPH_PRINT_IRQS) }, | ||
| 58 | { } /* Empty entry */ | 65 | { } /* Empty entry */ |
| 59 | }; | 66 | }; |
| 60 | 67 | ||
| 61 | static struct tracer_flags tracer_flags = { | 68 | static struct tracer_flags tracer_flags = { |
| 62 | /* Don't display overruns and proc by default */ | 69 | /* Don't display overruns and proc by default */ |
| 63 | .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD | | 70 | .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD | |
| 64 | TRACE_GRAPH_PRINT_DURATION, | 71 | TRACE_GRAPH_PRINT_DURATION | TRACE_GRAPH_PRINT_IRQS, |
| 65 | .opts = trace_opts | 72 | .opts = trace_opts |
| 66 | }; | 73 | }; |
| 67 | 74 | ||
| @@ -204,6 +211,14 @@ int __trace_graph_entry(struct trace_array *tr, | |||
| 204 | return 1; | 211 | return 1; |
| 205 | } | 212 | } |
| 206 | 213 | ||
| 214 | static inline int ftrace_graph_ignore_irqs(void) | ||
| 215 | { | ||
| 216 | if (!ftrace_graph_skip_irqs) | ||
| 217 | return 0; | ||
| 218 | |||
| 219 | return in_irq(); | ||
| 220 | } | ||
| 221 | |||
| 207 | int trace_graph_entry(struct ftrace_graph_ent *trace) | 222 | int trace_graph_entry(struct ftrace_graph_ent *trace) |
| 208 | { | 223 | { |
| 209 | struct trace_array *tr = graph_array; | 224 | struct trace_array *tr = graph_array; |
| @@ -218,7 +233,8 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) | |||
| 218 | return 0; | 233 | return 0; |
| 219 | 234 | ||
| 220 | /* trace it when it is-nested-in or is a function enabled. */ | 235 | /* trace it when it is-nested-in or is a function enabled. */ |
| 221 | if (!(trace->depth || ftrace_graph_addr(trace->func))) | 236 | if (!(trace->depth || ftrace_graph_addr(trace->func)) || |
| 237 | ftrace_graph_ignore_irqs()) | ||
| 222 | return 0; | 238 | return 0; |
| 223 | 239 | ||
| 224 | local_irq_save(flags); | 240 | local_irq_save(flags); |
| @@ -855,6 +871,92 @@ print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s, | |||
| 855 | return 0; | 871 | return 0; |
| 856 | } | 872 | } |
| 857 | 873 | ||
| 874 | /* | ||
| 875 | * Entry check for irq code | ||
| 876 | * | ||
| 877 | * returns 1 if | ||
| 878 | * - we are inside irq code | ||
| 879 | * - we just extered irq code | ||
| 880 | * | ||
| 881 | * retunns 0 if | ||
| 882 | * - funcgraph-interrupts option is set | ||
| 883 | * - we are not inside irq code | ||
| 884 | */ | ||
| 885 | static int | ||
| 886 | check_irq_entry(struct trace_iterator *iter, u32 flags, | ||
| 887 | unsigned long addr, int depth) | ||
| 888 | { | ||
| 889 | int cpu = iter->cpu; | ||
| 890 | struct fgraph_data *data = iter->private; | ||
| 891 | int *depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); | ||
| 892 | |||
| 893 | if (flags & TRACE_GRAPH_PRINT_IRQS) | ||
| 894 | return 0; | ||
| 895 | |||
| 896 | /* | ||
| 897 | * We are inside the irq code | ||
| 898 | */ | ||
| 899 | if (*depth_irq >= 0) | ||
| 900 | return 1; | ||
| 901 | |||
| 902 | if ((addr < (unsigned long)__irqentry_text_start) || | ||
| 903 | (addr >= (unsigned long)__irqentry_text_end)) | ||
| 904 | return 0; | ||
| 905 | |||
| 906 | /* | ||
| 907 | * We are entering irq code. | ||
| 908 | */ | ||
| 909 | *depth_irq = depth; | ||
| 910 | return 1; | ||
| 911 | } | ||
| 912 | |||
| 913 | /* | ||
| 914 | * Return check for irq code | ||
| 915 | * | ||
| 916 | * returns 1 if | ||
| 917 | * - we are inside irq code | ||
| 918 | * - we just left irq code | ||
| 919 | * | ||
| 920 | * returns 0 if | ||
| 921 | * - funcgraph-interrupts option is set | ||
| 922 | * - we are not inside irq code | ||
| 923 | */ | ||
| 924 | static int | ||
| 925 | check_irq_return(struct trace_iterator *iter, u32 flags, int depth) | ||
| 926 | { | ||
| 927 | int cpu = iter->cpu; | ||
| 928 | struct fgraph_data *data = iter->private; | ||
| 929 | int *depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); | ||
| 930 | |||
| 931 | if (flags & TRACE_GRAPH_PRINT_IRQS) | ||
| 932 | return 0; | ||
| 933 | |||
| 934 | /* | ||
| 935 | * We are not inside the irq code. | ||
| 936 | */ | ||
| 937 | if (*depth_irq == -1) | ||
| 938 | return 0; | ||
| 939 | |||
| 940 | /* | ||
| 941 | * We are inside the irq code, and this is returning entry. | ||
| 942 | * Let's not trace it and clear the entry depth, since | ||
| 943 | * we are out of irq code. | ||
| 944 | * | ||
| 945 | * This condition ensures that we 'leave the irq code' once | ||
| 946 | * we are out of the entry depth. Thus protecting us from | ||
| 947 | * the RETURN entry loss. | ||
| 948 | */ | ||
| 949 | if (*depth_irq >= depth) { | ||
| 950 | *depth_irq = -1; | ||
| 951 | return 1; | ||
| 952 | } | ||
| 953 | |||
| 954 | /* | ||
| 955 | * We are inside the irq code, and this is not the entry. | ||
| 956 | */ | ||
| 957 | return 1; | ||
| 958 | } | ||
| 959 | |||
| 858 | static enum print_line_t | 960 | static enum print_line_t |
| 859 | print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, | 961 | print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, |
| 860 | struct trace_iterator *iter, u32 flags) | 962 | struct trace_iterator *iter, u32 flags) |
| @@ -865,6 +967,9 @@ print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, | |||
| 865 | static enum print_line_t ret; | 967 | static enum print_line_t ret; |
| 866 | int cpu = iter->cpu; | 968 | int cpu = iter->cpu; |
| 867 | 969 | ||
| 970 | if (check_irq_entry(iter, flags, call->func, call->depth)) | ||
| 971 | return TRACE_TYPE_HANDLED; | ||
| 972 | |||
| 868 | if (print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func, flags)) | 973 | if (print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func, flags)) |
| 869 | return TRACE_TYPE_PARTIAL_LINE; | 974 | return TRACE_TYPE_PARTIAL_LINE; |
| 870 | 975 | ||
| @@ -902,6 +1007,9 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, | |||
| 902 | int ret; | 1007 | int ret; |
| 903 | int i; | 1008 | int i; |
| 904 | 1009 | ||
| 1010 | if (check_irq_return(iter, flags, trace->depth)) | ||
| 1011 | return TRACE_TYPE_HANDLED; | ||
| 1012 | |||
| 905 | if (data) { | 1013 | if (data) { |
| 906 | struct fgraph_cpu_data *cpu_data; | 1014 | struct fgraph_cpu_data *cpu_data; |
| 907 | int cpu = iter->cpu; | 1015 | int cpu = iter->cpu; |
| @@ -1210,9 +1318,12 @@ void graph_trace_open(struct trace_iterator *iter) | |||
| 1210 | pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); | 1318 | pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); |
| 1211 | int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth); | 1319 | int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth); |
| 1212 | int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore); | 1320 | int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore); |
| 1321 | int *depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); | ||
| 1322 | |||
| 1213 | *pid = -1; | 1323 | *pid = -1; |
| 1214 | *depth = 0; | 1324 | *depth = 0; |
| 1215 | *ignore = 0; | 1325 | *ignore = 0; |
| 1326 | *depth_irq = -1; | ||
| 1216 | } | 1327 | } |
| 1217 | 1328 | ||
| 1218 | iter->private = data; | 1329 | iter->private = data; |
| @@ -1235,6 +1346,14 @@ void graph_trace_close(struct trace_iterator *iter) | |||
| 1235 | } | 1346 | } |
| 1236 | } | 1347 | } |
| 1237 | 1348 | ||
| 1349 | static int func_graph_set_flag(u32 old_flags, u32 bit, int set) | ||
| 1350 | { | ||
| 1351 | if (bit == TRACE_GRAPH_PRINT_IRQS) | ||
| 1352 | ftrace_graph_skip_irqs = !set; | ||
| 1353 | |||
| 1354 | return 0; | ||
| 1355 | } | ||
| 1356 | |||
| 1238 | static struct trace_event_functions graph_functions = { | 1357 | static struct trace_event_functions graph_functions = { |
| 1239 | .trace = print_graph_function_event, | 1358 | .trace = print_graph_function_event, |
| 1240 | }; | 1359 | }; |
| @@ -1261,6 +1380,7 @@ static struct tracer graph_trace __read_mostly = { | |||
| 1261 | .print_line = print_graph_function, | 1380 | .print_line = print_graph_function, |
| 1262 | .print_header = print_graph_headers, | 1381 | .print_header = print_graph_headers, |
| 1263 | .flags = &tracer_flags, | 1382 | .flags = &tracer_flags, |
| 1383 | .set_flag = func_graph_set_flag, | ||
| 1264 | #ifdef CONFIG_FTRACE_SELFTEST | 1384 | #ifdef CONFIG_FTRACE_SELFTEST |
| 1265 | .selftest = trace_selftest_startup_function_graph, | 1385 | .selftest = trace_selftest_startup_function_graph, |
| 1266 | #endif | 1386 | #endif |
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c index a7cc3793baf6..209b379a4721 100644 --- a/kernel/trace/trace_workqueue.c +++ b/kernel/trace/trace_workqueue.c | |||
| @@ -263,6 +263,11 @@ int __init trace_workqueue_early_init(void) | |||
| 263 | { | 263 | { |
| 264 | int ret, cpu; | 264 | int ret, cpu; |
| 265 | 265 | ||
| 266 | for_each_possible_cpu(cpu) { | ||
| 267 | spin_lock_init(&workqueue_cpu_stat(cpu)->lock); | ||
| 268 | INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list); | ||
| 269 | } | ||
| 270 | |||
| 266 | ret = register_trace_workqueue_insertion(probe_workqueue_insertion, NULL); | 271 | ret = register_trace_workqueue_insertion(probe_workqueue_insertion, NULL); |
| 267 | if (ret) | 272 | if (ret) |
| 268 | goto out; | 273 | goto out; |
| @@ -279,11 +284,6 @@ int __init trace_workqueue_early_init(void) | |||
| 279 | if (ret) | 284 | if (ret) |
| 280 | goto no_creation; | 285 | goto no_creation; |
| 281 | 286 | ||
| 282 | for_each_possible_cpu(cpu) { | ||
| 283 | spin_lock_init(&workqueue_cpu_stat(cpu)->lock); | ||
| 284 | INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list); | ||
| 285 | } | ||
| 286 | |||
| 287 | return 0; | 287 | return 0; |
| 288 | 288 | ||
| 289 | no_creation: | 289 | no_creation: |
