diff options
author | Peter Zijlstra <a.p.zijlstra@chello.nl> | 2009-03-13 07:21:35 -0400 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2009-04-06 03:29:41 -0400 |
commit | d6d020e9957745c61285ef3da9f294c5e6801f0f (patch) | |
tree | efbd81871b58dbb026f19e812b224e1add2f3b76 | |
parent | ac17dc8e58f3069ea895cfff963adf98ff3cf6b2 (diff) |
perf_counter: hrtimer based sampling for software time events
Use hrtimers to profile timer based sampling for the software time
counters.
This allows platforms without hardware counter support to still
perform sample based profiling.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
-rw-r--r-- | include/linux/perf_counter.h | 20 | ||||
-rw-r--r-- | kernel/perf_counter.c | 123 |
2 files changed, 100 insertions, 43 deletions
diff --git a/include/linux/perf_counter.h b/include/linux/perf_counter.h index 4b14a8e9dbf5..dfb4c7ce18b3 100644 --- a/include/linux/perf_counter.h +++ b/include/linux/perf_counter.h | |||
@@ -114,6 +114,7 @@ struct perf_counter_hw_event { | |||
114 | #include <linux/rculist.h> | 114 | #include <linux/rculist.h> |
115 | #include <linux/rcupdate.h> | 115 | #include <linux/rcupdate.h> |
116 | #include <linux/spinlock.h> | 116 | #include <linux/spinlock.h> |
117 | #include <linux/hrtimer.h> | ||
117 | #include <asm/atomic.h> | 118 | #include <asm/atomic.h> |
118 | 119 | ||
119 | struct task_struct; | 120 | struct task_struct; |
@@ -123,12 +124,19 @@ struct task_struct; | |||
123 | */ | 124 | */ |
124 | struct hw_perf_counter { | 125 | struct hw_perf_counter { |
125 | #ifdef CONFIG_PERF_COUNTERS | 126 | #ifdef CONFIG_PERF_COUNTERS |
126 | u64 config; | 127 | union { |
127 | unsigned long config_base; | 128 | struct { /* hardware */ |
128 | unsigned long counter_base; | 129 | u64 config; |
129 | int nmi; | 130 | unsigned long config_base; |
130 | unsigned int idx; | 131 | unsigned long counter_base; |
131 | atomic64_t count; /* software */ | 132 | int nmi; |
133 | unsigned int idx; | ||
134 | }; | ||
135 | union { /* software */ | ||
136 | atomic64_t count; | ||
137 | struct hrtimer hrtimer; | ||
138 | }; | ||
139 | }; | ||
132 | atomic64_t prev_count; | 140 | atomic64_t prev_count; |
133 | u64 irq_period; | 141 | u64 irq_period; |
134 | atomic64_t period_left; | 142 | atomic64_t period_left; |
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c index 68950a3a52bf..f9330d5827cf 100644 --- a/kernel/perf_counter.c +++ b/kernel/perf_counter.c | |||
@@ -1395,7 +1395,7 @@ static void perf_swcounter_handle_group(struct perf_counter *sibling) | |||
1395 | struct perf_counter *counter, *group_leader = sibling->group_leader; | 1395 | struct perf_counter *counter, *group_leader = sibling->group_leader; |
1396 | 1396 | ||
1397 | list_for_each_entry(counter, &group_leader->sibling_list, list_entry) { | 1397 | list_for_each_entry(counter, &group_leader->sibling_list, list_entry) { |
1398 | perf_swcounter_update(counter); | 1398 | counter->hw_ops->read(counter); |
1399 | perf_swcounter_store_irq(sibling, counter->hw_event.type); | 1399 | perf_swcounter_store_irq(sibling, counter->hw_event.type); |
1400 | perf_swcounter_store_irq(sibling, atomic64_read(&counter->count)); | 1400 | perf_swcounter_store_irq(sibling, atomic64_read(&counter->count)); |
1401 | } | 1401 | } |
@@ -1404,8 +1404,6 @@ static void perf_swcounter_handle_group(struct perf_counter *sibling) | |||
1404 | static void perf_swcounter_interrupt(struct perf_counter *counter, | 1404 | static void perf_swcounter_interrupt(struct perf_counter *counter, |
1405 | int nmi, struct pt_regs *regs) | 1405 | int nmi, struct pt_regs *regs) |
1406 | { | 1406 | { |
1407 | perf_swcounter_save_and_restart(counter); | ||
1408 | |||
1409 | switch (counter->hw_event.record_type) { | 1407 | switch (counter->hw_event.record_type) { |
1410 | case PERF_RECORD_SIMPLE: | 1408 | case PERF_RECORD_SIMPLE: |
1411 | break; | 1409 | break; |
@@ -1426,6 +1424,38 @@ static void perf_swcounter_interrupt(struct perf_counter *counter, | |||
1426 | wake_up(&counter->waitq); | 1424 | wake_up(&counter->waitq); |
1427 | } | 1425 | } |
1428 | 1426 | ||
1427 | static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer) | ||
1428 | { | ||
1429 | struct perf_counter *counter; | ||
1430 | struct pt_regs *regs; | ||
1431 | |||
1432 | counter = container_of(hrtimer, struct perf_counter, hw.hrtimer); | ||
1433 | counter->hw_ops->read(counter); | ||
1434 | |||
1435 | regs = get_irq_regs(); | ||
1436 | /* | ||
1437 | * In case we exclude kernel IPs or are somehow not in interrupt | ||
1438 | * context, provide the next best thing, the user IP. | ||
1439 | */ | ||
1440 | if ((counter->hw_event.exclude_kernel || !regs) && | ||
1441 | !counter->hw_event.exclude_user) | ||
1442 | regs = task_pt_regs(current); | ||
1443 | |||
1444 | if (regs) | ||
1445 | perf_swcounter_interrupt(counter, 0, regs); | ||
1446 | |||
1447 | hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period)); | ||
1448 | |||
1449 | return HRTIMER_RESTART; | ||
1450 | } | ||
1451 | |||
1452 | static void perf_swcounter_overflow(struct perf_counter *counter, | ||
1453 | int nmi, struct pt_regs *regs) | ||
1454 | { | ||
1455 | perf_swcounter_save_and_restart(counter); | ||
1456 | perf_swcounter_interrupt(counter, nmi, regs); | ||
1457 | } | ||
1458 | |||
1429 | static int perf_swcounter_match(struct perf_counter *counter, | 1459 | static int perf_swcounter_match(struct perf_counter *counter, |
1430 | enum hw_event_types event, | 1460 | enum hw_event_types event, |
1431 | struct pt_regs *regs) | 1461 | struct pt_regs *regs) |
@@ -1448,13 +1478,20 @@ static int perf_swcounter_match(struct perf_counter *counter, | |||
1448 | return 1; | 1478 | return 1; |
1449 | } | 1479 | } |
1450 | 1480 | ||
1481 | static void perf_swcounter_add(struct perf_counter *counter, u64 nr, | ||
1482 | int nmi, struct pt_regs *regs) | ||
1483 | { | ||
1484 | int neg = atomic64_add_negative(nr, &counter->hw.count); | ||
1485 | if (counter->hw.irq_period && !neg) | ||
1486 | perf_swcounter_overflow(counter, nmi, regs); | ||
1487 | } | ||
1488 | |||
1451 | static void perf_swcounter_ctx_event(struct perf_counter_context *ctx, | 1489 | static void perf_swcounter_ctx_event(struct perf_counter_context *ctx, |
1452 | enum hw_event_types event, u64 nr, | 1490 | enum hw_event_types event, u64 nr, |
1453 | int nmi, struct pt_regs *regs) | 1491 | int nmi, struct pt_regs *regs) |
1454 | { | 1492 | { |
1455 | struct perf_counter *counter; | 1493 | struct perf_counter *counter; |
1456 | unsigned long flags; | 1494 | unsigned long flags; |
1457 | int neg; | ||
1458 | 1495 | ||
1459 | if (list_empty(&ctx->counter_list)) | 1496 | if (list_empty(&ctx->counter_list)) |
1460 | return; | 1497 | return; |
@@ -1465,11 +1502,8 @@ static void perf_swcounter_ctx_event(struct perf_counter_context *ctx, | |||
1465 | * XXX: make counter_list RCU safe | 1502 | * XXX: make counter_list RCU safe |
1466 | */ | 1503 | */ |
1467 | list_for_each_entry(counter, &ctx->counter_list, list_entry) { | 1504 | list_for_each_entry(counter, &ctx->counter_list, list_entry) { |
1468 | if (perf_swcounter_match(counter, event, regs)) { | 1505 | if (perf_swcounter_match(counter, event, regs)) |
1469 | neg = atomic64_add_negative(nr, &counter->hw.count); | 1506 | perf_swcounter_add(counter, nr, nmi, regs); |
1470 | if (counter->hw.irq_period && !neg) | ||
1471 | perf_swcounter_interrupt(counter, nmi, regs); | ||
1472 | } | ||
1473 | } | 1507 | } |
1474 | 1508 | ||
1475 | spin_unlock_irqrestore(&ctx->lock, flags); | 1509 | spin_unlock_irqrestore(&ctx->lock, flags); |
@@ -1513,14 +1547,6 @@ static const struct hw_perf_counter_ops perf_ops_generic = { | |||
1513 | * Software counter: cpu wall time clock | 1547 | * Software counter: cpu wall time clock |
1514 | */ | 1548 | */ |
1515 | 1549 | ||
1516 | static int cpu_clock_perf_counter_enable(struct perf_counter *counter) | ||
1517 | { | ||
1518 | int cpu = raw_smp_processor_id(); | ||
1519 | |||
1520 | atomic64_set(&counter->hw.prev_count, cpu_clock(cpu)); | ||
1521 | return 0; | ||
1522 | } | ||
1523 | |||
1524 | static void cpu_clock_perf_counter_update(struct perf_counter *counter) | 1550 | static void cpu_clock_perf_counter_update(struct perf_counter *counter) |
1525 | { | 1551 | { |
1526 | int cpu = raw_smp_processor_id(); | 1552 | int cpu = raw_smp_processor_id(); |
@@ -1533,8 +1559,26 @@ static void cpu_clock_perf_counter_update(struct perf_counter *counter) | |||
1533 | atomic64_add(now - prev, &counter->count); | 1559 | atomic64_add(now - prev, &counter->count); |
1534 | } | 1560 | } |
1535 | 1561 | ||
1562 | static int cpu_clock_perf_counter_enable(struct perf_counter *counter) | ||
1563 | { | ||
1564 | struct hw_perf_counter *hwc = &counter->hw; | ||
1565 | int cpu = raw_smp_processor_id(); | ||
1566 | |||
1567 | atomic64_set(&hwc->prev_count, cpu_clock(cpu)); | ||
1568 | if (hwc->irq_period) { | ||
1569 | hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | ||
1570 | hwc->hrtimer.function = perf_swcounter_hrtimer; | ||
1571 | __hrtimer_start_range_ns(&hwc->hrtimer, | ||
1572 | ns_to_ktime(hwc->irq_period), 0, | ||
1573 | HRTIMER_MODE_REL, 0); | ||
1574 | } | ||
1575 | |||
1576 | return 0; | ||
1577 | } | ||
1578 | |||
1536 | static void cpu_clock_perf_counter_disable(struct perf_counter *counter) | 1579 | static void cpu_clock_perf_counter_disable(struct perf_counter *counter) |
1537 | { | 1580 | { |
1581 | hrtimer_cancel(&counter->hw.hrtimer); | ||
1538 | cpu_clock_perf_counter_update(counter); | 1582 | cpu_clock_perf_counter_update(counter); |
1539 | } | 1583 | } |
1540 | 1584 | ||
@@ -1580,27 +1624,33 @@ static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now | |||
1580 | atomic64_add(delta, &counter->count); | 1624 | atomic64_add(delta, &counter->count); |
1581 | } | 1625 | } |
1582 | 1626 | ||
1583 | static void task_clock_perf_counter_read(struct perf_counter *counter) | ||
1584 | { | ||
1585 | u64 now = task_clock_perf_counter_val(counter, 1); | ||
1586 | |||
1587 | task_clock_perf_counter_update(counter, now); | ||
1588 | } | ||
1589 | |||
1590 | static int task_clock_perf_counter_enable(struct perf_counter *counter) | 1627 | static int task_clock_perf_counter_enable(struct perf_counter *counter) |
1591 | { | 1628 | { |
1592 | if (counter->prev_state <= PERF_COUNTER_STATE_OFF) | 1629 | struct hw_perf_counter *hwc = &counter->hw; |
1593 | atomic64_set(&counter->hw.prev_count, | 1630 | |
1594 | task_clock_perf_counter_val(counter, 0)); | 1631 | atomic64_set(&hwc->prev_count, task_clock_perf_counter_val(counter, 0)); |
1632 | if (hwc->irq_period) { | ||
1633 | hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | ||
1634 | hwc->hrtimer.function = perf_swcounter_hrtimer; | ||
1635 | __hrtimer_start_range_ns(&hwc->hrtimer, | ||
1636 | ns_to_ktime(hwc->irq_period), 0, | ||
1637 | HRTIMER_MODE_REL, 0); | ||
1638 | } | ||
1595 | 1639 | ||
1596 | return 0; | 1640 | return 0; |
1597 | } | 1641 | } |
1598 | 1642 | ||
1599 | static void task_clock_perf_counter_disable(struct perf_counter *counter) | 1643 | static void task_clock_perf_counter_disable(struct perf_counter *counter) |
1600 | { | 1644 | { |
1601 | u64 now = task_clock_perf_counter_val(counter, 0); | 1645 | hrtimer_cancel(&counter->hw.hrtimer); |
1646 | task_clock_perf_counter_update(counter, | ||
1647 | task_clock_perf_counter_val(counter, 0)); | ||
1648 | } | ||
1602 | 1649 | ||
1603 | task_clock_perf_counter_update(counter, now); | 1650 | static void task_clock_perf_counter_read(struct perf_counter *counter) |
1651 | { | ||
1652 | task_clock_perf_counter_update(counter, | ||
1653 | task_clock_perf_counter_val(counter, 1)); | ||
1604 | } | 1654 | } |
1605 | 1655 | ||
1606 | static const struct hw_perf_counter_ops perf_ops_task_clock = { | 1656 | static const struct hw_perf_counter_ops perf_ops_task_clock = { |
@@ -1729,16 +1779,12 @@ sw_perf_counter_init(struct perf_counter *counter) | |||
1729 | */ | 1779 | */ |
1730 | switch (counter->hw_event.type) { | 1780 | switch (counter->hw_event.type) { |
1731 | case PERF_COUNT_CPU_CLOCK: | 1781 | case PERF_COUNT_CPU_CLOCK: |
1732 | if (!(counter->hw_event.exclude_user || | 1782 | hw_ops = &perf_ops_cpu_clock; |
1733 | counter->hw_event.exclude_kernel || | 1783 | |
1734 | counter->hw_event.exclude_hv)) | 1784 | if (hw_event->irq_period && hw_event->irq_period < 10000) |
1735 | hw_ops = &perf_ops_cpu_clock; | 1785 | hw_event->irq_period = 10000; |
1736 | break; | 1786 | break; |
1737 | case PERF_COUNT_TASK_CLOCK: | 1787 | case PERF_COUNT_TASK_CLOCK: |
1738 | if (counter->hw_event.exclude_user || | ||
1739 | counter->hw_event.exclude_kernel || | ||
1740 | counter->hw_event.exclude_hv) | ||
1741 | break; | ||
1742 | /* | 1788 | /* |
1743 | * If the user instantiates this as a per-cpu counter, | 1789 | * If the user instantiates this as a per-cpu counter, |
1744 | * use the cpu_clock counter instead. | 1790 | * use the cpu_clock counter instead. |
@@ -1747,6 +1793,9 @@ sw_perf_counter_init(struct perf_counter *counter) | |||
1747 | hw_ops = &perf_ops_task_clock; | 1793 | hw_ops = &perf_ops_task_clock; |
1748 | else | 1794 | else |
1749 | hw_ops = &perf_ops_cpu_clock; | 1795 | hw_ops = &perf_ops_cpu_clock; |
1796 | |||
1797 | if (hw_event->irq_period && hw_event->irq_period < 10000) | ||
1798 | hw_event->irq_period = 10000; | ||
1750 | break; | 1799 | break; |
1751 | case PERF_COUNT_PAGE_FAULTS: | 1800 | case PERF_COUNT_PAGE_FAULTS: |
1752 | case PERF_COUNT_PAGE_FAULTS_MIN: | 1801 | case PERF_COUNT_PAGE_FAULTS_MIN: |