diff options
Diffstat (limited to 'arch/x86/oprofile')
-rw-r--r-- | arch/x86/oprofile/backtrace.c | 83 | ||||
-rw-r--r-- | arch/x86/oprofile/nmi_int.c | 84 | ||||
-rw-r--r-- | arch/x86/oprofile/nmi_timer_int.c | 5 | ||||
-rw-r--r-- | arch/x86/oprofile/op_counter.h | 1 | ||||
-rw-r--r-- | arch/x86/oprofile/op_model_amd.c | 356 | ||||
-rw-r--r-- | arch/x86/oprofile/op_model_p4.c | 4 | ||||
-rw-r--r-- | arch/x86/oprofile/op_model_ppro.c | 8 |
7 files changed, 381 insertions, 160 deletions
diff --git a/arch/x86/oprofile/backtrace.c b/arch/x86/oprofile/backtrace.c index 3855096c59b8..a5b64ab4cd6e 100644 --- a/arch/x86/oprofile/backtrace.c +++ b/arch/x86/oprofile/backtrace.c | |||
@@ -14,17 +14,7 @@ | |||
14 | #include <asm/ptrace.h> | 14 | #include <asm/ptrace.h> |
15 | #include <asm/uaccess.h> | 15 | #include <asm/uaccess.h> |
16 | #include <asm/stacktrace.h> | 16 | #include <asm/stacktrace.h> |
17 | 17 | #include <linux/compat.h> | |
18 | static void backtrace_warning_symbol(void *data, char *msg, | ||
19 | unsigned long symbol) | ||
20 | { | ||
21 | /* Ignore warnings */ | ||
22 | } | ||
23 | |||
24 | static void backtrace_warning(void *data, char *msg) | ||
25 | { | ||
26 | /* Ignore warnings */ | ||
27 | } | ||
28 | 18 | ||
29 | static int backtrace_stack(void *data, char *name) | 19 | static int backtrace_stack(void *data, char *name) |
30 | { | 20 | { |
@@ -41,21 +31,17 @@ static void backtrace_address(void *data, unsigned long addr, int reliable) | |||
41 | } | 31 | } |
42 | 32 | ||
43 | static struct stacktrace_ops backtrace_ops = { | 33 | static struct stacktrace_ops backtrace_ops = { |
44 | .warning = backtrace_warning, | ||
45 | .warning_symbol = backtrace_warning_symbol, | ||
46 | .stack = backtrace_stack, | 34 | .stack = backtrace_stack, |
47 | .address = backtrace_address, | 35 | .address = backtrace_address, |
48 | .walk_stack = print_context_stack, | 36 | .walk_stack = print_context_stack, |
49 | }; | 37 | }; |
50 | 38 | ||
51 | struct frame_head { | 39 | #ifdef CONFIG_COMPAT |
52 | struct frame_head *bp; | 40 | static struct stack_frame_ia32 * |
53 | unsigned long ret; | 41 | dump_user_backtrace_32(struct stack_frame_ia32 *head) |
54 | } __attribute__((packed)); | ||
55 | |||
56 | static struct frame_head *dump_user_backtrace(struct frame_head *head) | ||
57 | { | 42 | { |
58 | struct frame_head bufhead[2]; | 43 | struct stack_frame_ia32 bufhead[2]; |
44 | struct stack_frame_ia32 *fp; | ||
59 | 45 | ||
60 | /* Also check accessibility of one struct frame_head beyond */ | 46 | /* Also check accessibility of one struct frame_head beyond */ |
61 | if (!access_ok(VERIFY_READ, head, sizeof(bufhead))) | 47 | if (!access_ok(VERIFY_READ, head, sizeof(bufhead))) |
@@ -63,20 +49,66 @@ static struct frame_head *dump_user_backtrace(struct frame_head *head) | |||
63 | if (__copy_from_user_inatomic(bufhead, head, sizeof(bufhead))) | 49 | if (__copy_from_user_inatomic(bufhead, head, sizeof(bufhead))) |
64 | return NULL; | 50 | return NULL; |
65 | 51 | ||
66 | oprofile_add_trace(bufhead[0].ret); | 52 | fp = (struct stack_frame_ia32 *) compat_ptr(bufhead[0].next_frame); |
53 | |||
54 | oprofile_add_trace(bufhead[0].return_address); | ||
55 | |||
56 | /* frame pointers should strictly progress back up the stack | ||
57 | * (towards higher addresses) */ | ||
58 | if (head >= fp) | ||
59 | return NULL; | ||
60 | |||
61 | return fp; | ||
62 | } | ||
63 | |||
64 | static inline int | ||
65 | x86_backtrace_32(struct pt_regs * const regs, unsigned int depth) | ||
66 | { | ||
67 | struct stack_frame_ia32 *head; | ||
68 | |||
69 | /* User process is 32-bit */ | ||
70 | if (!current || !test_thread_flag(TIF_IA32)) | ||
71 | return 0; | ||
72 | |||
73 | head = (struct stack_frame_ia32 *) regs->bp; | ||
74 | while (depth-- && head) | ||
75 | head = dump_user_backtrace_32(head); | ||
76 | |||
77 | return 1; | ||
78 | } | ||
79 | |||
80 | #else | ||
81 | static inline int | ||
82 | x86_backtrace_32(struct pt_regs * const regs, unsigned int depth) | ||
83 | { | ||
84 | return 0; | ||
85 | } | ||
86 | #endif /* CONFIG_COMPAT */ | ||
87 | |||
88 | static struct stack_frame *dump_user_backtrace(struct stack_frame *head) | ||
89 | { | ||
90 | struct stack_frame bufhead[2]; | ||
91 | |||
92 | /* Also check accessibility of one struct stack_frame beyond */ | ||
93 | if (!access_ok(VERIFY_READ, head, sizeof(bufhead))) | ||
94 | return NULL; | ||
95 | if (__copy_from_user_inatomic(bufhead, head, sizeof(bufhead))) | ||
96 | return NULL; | ||
97 | |||
98 | oprofile_add_trace(bufhead[0].return_address); | ||
67 | 99 | ||
68 | /* frame pointers should strictly progress back up the stack | 100 | /* frame pointers should strictly progress back up the stack |
69 | * (towards higher addresses) */ | 101 | * (towards higher addresses) */ |
70 | if (head >= bufhead[0].bp) | 102 | if (head >= bufhead[0].next_frame) |
71 | return NULL; | 103 | return NULL; |
72 | 104 | ||
73 | return bufhead[0].bp; | 105 | return bufhead[0].next_frame; |
74 | } | 106 | } |
75 | 107 | ||
76 | void | 108 | void |
77 | x86_backtrace(struct pt_regs * const regs, unsigned int depth) | 109 | x86_backtrace(struct pt_regs * const regs, unsigned int depth) |
78 | { | 110 | { |
79 | struct frame_head *head = (struct frame_head *)frame_pointer(regs); | 111 | struct stack_frame *head = (struct stack_frame *)frame_pointer(regs); |
80 | 112 | ||
81 | if (!user_mode_vm(regs)) { | 113 | if (!user_mode_vm(regs)) { |
82 | unsigned long stack = kernel_stack_pointer(regs); | 114 | unsigned long stack = kernel_stack_pointer(regs); |
@@ -86,6 +118,9 @@ x86_backtrace(struct pt_regs * const regs, unsigned int depth) | |||
86 | return; | 118 | return; |
87 | } | 119 | } |
88 | 120 | ||
121 | if (x86_backtrace_32(regs, depth)) | ||
122 | return; | ||
123 | |||
89 | while (depth-- && head) | 124 | while (depth-- && head) |
90 | head = dump_user_backtrace(head); | 125 | head = dump_user_backtrace(head); |
91 | } | 126 | } |
diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c index f1575c9a2572..68894fdc034b 100644 --- a/arch/x86/oprofile/nmi_int.c +++ b/arch/x86/oprofile/nmi_int.c | |||
@@ -15,7 +15,7 @@ | |||
15 | #include <linux/notifier.h> | 15 | #include <linux/notifier.h> |
16 | #include <linux/smp.h> | 16 | #include <linux/smp.h> |
17 | #include <linux/oprofile.h> | 17 | #include <linux/oprofile.h> |
18 | #include <linux/sysdev.h> | 18 | #include <linux/syscore_ops.h> |
19 | #include <linux/slab.h> | 19 | #include <linux/slab.h> |
20 | #include <linux/moduleparam.h> | 20 | #include <linux/moduleparam.h> |
21 | #include <linux/kdebug.h> | 21 | #include <linux/kdebug.h> |
@@ -49,6 +49,10 @@ u64 op_x86_get_ctrl(struct op_x86_model_spec const *model, | |||
49 | val |= counter_config->user ? ARCH_PERFMON_EVENTSEL_USR : 0; | 49 | val |= counter_config->user ? ARCH_PERFMON_EVENTSEL_USR : 0; |
50 | val |= counter_config->kernel ? ARCH_PERFMON_EVENTSEL_OS : 0; | 50 | val |= counter_config->kernel ? ARCH_PERFMON_EVENTSEL_OS : 0; |
51 | val |= (counter_config->unit_mask & 0xFF) << 8; | 51 | val |= (counter_config->unit_mask & 0xFF) << 8; |
52 | counter_config->extra &= (ARCH_PERFMON_EVENTSEL_INV | | ||
53 | ARCH_PERFMON_EVENTSEL_EDGE | | ||
54 | ARCH_PERFMON_EVENTSEL_CMASK); | ||
55 | val |= counter_config->extra; | ||
52 | event &= model->event_mask ? model->event_mask : 0xFF; | 56 | event &= model->event_mask ? model->event_mask : 0xFF; |
53 | val |= event & 0xFF; | 57 | val |= event & 0xFF; |
54 | val |= (event & 0x0F00) << 24; | 58 | val |= (event & 0x0F00) << 24; |
@@ -65,7 +69,6 @@ static int profile_exceptions_notify(struct notifier_block *self, | |||
65 | 69 | ||
66 | switch (val) { | 70 | switch (val) { |
67 | case DIE_NMI: | 71 | case DIE_NMI: |
68 | case DIE_NMI_IPI: | ||
69 | if (ctr_running) | 72 | if (ctr_running) |
70 | model->check_ctrs(args->regs, &__get_cpu_var(cpu_msrs)); | 73 | model->check_ctrs(args->regs, &__get_cpu_var(cpu_msrs)); |
71 | else if (!nmi_enabled) | 74 | else if (!nmi_enabled) |
@@ -109,8 +112,10 @@ static void nmi_cpu_start(void *dummy) | |||
109 | static int nmi_start(void) | 112 | static int nmi_start(void) |
110 | { | 113 | { |
111 | get_online_cpus(); | 114 | get_online_cpus(); |
112 | on_each_cpu(nmi_cpu_start, NULL, 1); | ||
113 | ctr_running = 1; | 115 | ctr_running = 1; |
116 | /* make ctr_running visible to the nmi handler: */ | ||
117 | smp_mb(); | ||
118 | on_each_cpu(nmi_cpu_start, NULL, 1); | ||
114 | put_online_cpus(); | 119 | put_online_cpus(); |
115 | return 0; | 120 | return 0; |
116 | } | 121 | } |
@@ -143,7 +148,7 @@ static inline int has_mux(void) | |||
143 | 148 | ||
144 | inline int op_x86_phys_to_virt(int phys) | 149 | inline int op_x86_phys_to_virt(int phys) |
145 | { | 150 | { |
146 | return __get_cpu_var(switch_index) + phys; | 151 | return __this_cpu_read(switch_index) + phys; |
147 | } | 152 | } |
148 | 153 | ||
149 | inline int op_x86_virt_to_phys(int virt) | 154 | inline int op_x86_virt_to_phys(int virt) |
@@ -361,7 +366,7 @@ static void nmi_cpu_setup(void *dummy) | |||
361 | static struct notifier_block profile_exceptions_nb = { | 366 | static struct notifier_block profile_exceptions_nb = { |
362 | .notifier_call = profile_exceptions_notify, | 367 | .notifier_call = profile_exceptions_notify, |
363 | .next = NULL, | 368 | .next = NULL, |
364 | .priority = 2 | 369 | .priority = NMI_LOCAL_LOW_PRIOR, |
365 | }; | 370 | }; |
366 | 371 | ||
367 | static void nmi_cpu_restore_registers(struct op_msrs *msrs) | 372 | static void nmi_cpu_restore_registers(struct op_msrs *msrs) |
@@ -441,6 +446,7 @@ static int nmi_create_files(struct super_block *sb, struct dentry *root) | |||
441 | oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config[i].unit_mask); | 446 | oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config[i].unit_mask); |
442 | oprofilefs_create_ulong(sb, dir, "kernel", &counter_config[i].kernel); | 447 | oprofilefs_create_ulong(sb, dir, "kernel", &counter_config[i].kernel); |
443 | oprofilefs_create_ulong(sb, dir, "user", &counter_config[i].user); | 448 | oprofilefs_create_ulong(sb, dir, "user", &counter_config[i].user); |
449 | oprofilefs_create_ulong(sb, dir, "extra", &counter_config[i].extra); | ||
444 | } | 450 | } |
445 | 451 | ||
446 | return 0; | 452 | return 0; |
@@ -500,15 +506,18 @@ static int nmi_setup(void) | |||
500 | 506 | ||
501 | nmi_enabled = 0; | 507 | nmi_enabled = 0; |
502 | ctr_running = 0; | 508 | ctr_running = 0; |
503 | barrier(); | 509 | /* make variables visible to the nmi handler: */ |
510 | smp_mb(); | ||
504 | err = register_die_notifier(&profile_exceptions_nb); | 511 | err = register_die_notifier(&profile_exceptions_nb); |
505 | if (err) | 512 | if (err) |
506 | goto fail; | 513 | goto fail; |
507 | 514 | ||
508 | get_online_cpus(); | 515 | get_online_cpus(); |
509 | register_cpu_notifier(&oprofile_cpu_nb); | 516 | register_cpu_notifier(&oprofile_cpu_nb); |
510 | on_each_cpu(nmi_cpu_setup, NULL, 1); | ||
511 | nmi_enabled = 1; | 517 | nmi_enabled = 1; |
518 | /* make nmi_enabled visible to the nmi handler: */ | ||
519 | smp_mb(); | ||
520 | on_each_cpu(nmi_cpu_setup, NULL, 1); | ||
512 | put_online_cpus(); | 521 | put_online_cpus(); |
513 | 522 | ||
514 | return 0; | 523 | return 0; |
@@ -527,7 +536,8 @@ static void nmi_shutdown(void) | |||
527 | nmi_enabled = 0; | 536 | nmi_enabled = 0; |
528 | ctr_running = 0; | 537 | ctr_running = 0; |
529 | put_online_cpus(); | 538 | put_online_cpus(); |
530 | barrier(); | 539 | /* make variables visible to the nmi handler: */ |
540 | smp_mb(); | ||
531 | unregister_die_notifier(&profile_exceptions_nb); | 541 | unregister_die_notifier(&profile_exceptions_nb); |
532 | msrs = &get_cpu_var(cpu_msrs); | 542 | msrs = &get_cpu_var(cpu_msrs); |
533 | model->shutdown(msrs); | 543 | model->shutdown(msrs); |
@@ -537,7 +547,7 @@ static void nmi_shutdown(void) | |||
537 | 547 | ||
538 | #ifdef CONFIG_PM | 548 | #ifdef CONFIG_PM |
539 | 549 | ||
540 | static int nmi_suspend(struct sys_device *dev, pm_message_t state) | 550 | static int nmi_suspend(void) |
541 | { | 551 | { |
542 | /* Only one CPU left, just stop that one */ | 552 | /* Only one CPU left, just stop that one */ |
543 | if (nmi_enabled == 1) | 553 | if (nmi_enabled == 1) |
@@ -545,49 +555,31 @@ static int nmi_suspend(struct sys_device *dev, pm_message_t state) | |||
545 | return 0; | 555 | return 0; |
546 | } | 556 | } |
547 | 557 | ||
548 | static int nmi_resume(struct sys_device *dev) | 558 | static void nmi_resume(void) |
549 | { | 559 | { |
550 | if (nmi_enabled == 1) | 560 | if (nmi_enabled == 1) |
551 | nmi_cpu_start(NULL); | 561 | nmi_cpu_start(NULL); |
552 | return 0; | ||
553 | } | 562 | } |
554 | 563 | ||
555 | static struct sysdev_class oprofile_sysclass = { | 564 | static struct syscore_ops oprofile_syscore_ops = { |
556 | .name = "oprofile", | ||
557 | .resume = nmi_resume, | 565 | .resume = nmi_resume, |
558 | .suspend = nmi_suspend, | 566 | .suspend = nmi_suspend, |
559 | }; | 567 | }; |
560 | 568 | ||
561 | static struct sys_device device_oprofile = { | 569 | static void __init init_suspend_resume(void) |
562 | .id = 0, | ||
563 | .cls = &oprofile_sysclass, | ||
564 | }; | ||
565 | |||
566 | static int __init init_sysfs(void) | ||
567 | { | 570 | { |
568 | int error; | 571 | register_syscore_ops(&oprofile_syscore_ops); |
569 | |||
570 | error = sysdev_class_register(&oprofile_sysclass); | ||
571 | if (error) | ||
572 | return error; | ||
573 | |||
574 | error = sysdev_register(&device_oprofile); | ||
575 | if (error) | ||
576 | sysdev_class_unregister(&oprofile_sysclass); | ||
577 | |||
578 | return error; | ||
579 | } | 572 | } |
580 | 573 | ||
581 | static void exit_sysfs(void) | 574 | static void exit_suspend_resume(void) |
582 | { | 575 | { |
583 | sysdev_unregister(&device_oprofile); | 576 | unregister_syscore_ops(&oprofile_syscore_ops); |
584 | sysdev_class_unregister(&oprofile_sysclass); | ||
585 | } | 577 | } |
586 | 578 | ||
587 | #else | 579 | #else |
588 | 580 | ||
589 | static inline int init_sysfs(void) { return 0; } | 581 | static inline void init_suspend_resume(void) { } |
590 | static inline void exit_sysfs(void) { } | 582 | static inline void exit_suspend_resume(void) { } |
591 | 583 | ||
592 | #endif /* CONFIG_PM */ | 584 | #endif /* CONFIG_PM */ |
593 | 585 | ||
@@ -695,9 +687,6 @@ static int __init ppro_init(char **cpu_type) | |||
695 | return 1; | 687 | return 1; |
696 | } | 688 | } |
697 | 689 | ||
698 | /* in order to get sysfs right */ | ||
699 | static int using_nmi; | ||
700 | |||
701 | int __init op_nmi_init(struct oprofile_operations *ops) | 690 | int __init op_nmi_init(struct oprofile_operations *ops) |
702 | { | 691 | { |
703 | __u8 vendor = boot_cpu_data.x86_vendor; | 692 | __u8 vendor = boot_cpu_data.x86_vendor; |
@@ -705,8 +694,6 @@ int __init op_nmi_init(struct oprofile_operations *ops) | |||
705 | char *cpu_type = NULL; | 694 | char *cpu_type = NULL; |
706 | int ret = 0; | 695 | int ret = 0; |
707 | 696 | ||
708 | using_nmi = 0; | ||
709 | |||
710 | if (!cpu_has_apic) | 697 | if (!cpu_has_apic) |
711 | return -ENODEV; | 698 | return -ENODEV; |
712 | 699 | ||
@@ -731,6 +718,15 @@ int __init op_nmi_init(struct oprofile_operations *ops) | |||
731 | case 0x11: | 718 | case 0x11: |
732 | cpu_type = "x86-64/family11h"; | 719 | cpu_type = "x86-64/family11h"; |
733 | break; | 720 | break; |
721 | case 0x12: | ||
722 | cpu_type = "x86-64/family12h"; | ||
723 | break; | ||
724 | case 0x14: | ||
725 | cpu_type = "x86-64/family14h"; | ||
726 | break; | ||
727 | case 0x15: | ||
728 | cpu_type = "x86-64/family15h"; | ||
729 | break; | ||
734 | default: | 730 | default: |
735 | return -ENODEV; | 731 | return -ENODEV; |
736 | } | 732 | } |
@@ -786,17 +782,13 @@ int __init op_nmi_init(struct oprofile_operations *ops) | |||
786 | 782 | ||
787 | mux_init(ops); | 783 | mux_init(ops); |
788 | 784 | ||
789 | ret = init_sysfs(); | 785 | init_suspend_resume(); |
790 | if (ret) | ||
791 | return ret; | ||
792 | 786 | ||
793 | using_nmi = 1; | ||
794 | printk(KERN_INFO "oprofile: using NMI interrupt.\n"); | 787 | printk(KERN_INFO "oprofile: using NMI interrupt.\n"); |
795 | return 0; | 788 | return 0; |
796 | } | 789 | } |
797 | 790 | ||
798 | void op_nmi_exit(void) | 791 | void op_nmi_exit(void) |
799 | { | 792 | { |
800 | if (using_nmi) | 793 | exit_suspend_resume(); |
801 | exit_sysfs(); | ||
802 | } | 794 | } |
diff --git a/arch/x86/oprofile/nmi_timer_int.c b/arch/x86/oprofile/nmi_timer_int.c index e3ecb71b5790..720bf5a53c51 100644 --- a/arch/x86/oprofile/nmi_timer_int.c +++ b/arch/x86/oprofile/nmi_timer_int.c | |||
@@ -38,7 +38,7 @@ static int profile_timer_exceptions_notify(struct notifier_block *self, | |||
38 | static struct notifier_block profile_timer_exceptions_nb = { | 38 | static struct notifier_block profile_timer_exceptions_nb = { |
39 | .notifier_call = profile_timer_exceptions_notify, | 39 | .notifier_call = profile_timer_exceptions_notify, |
40 | .next = NULL, | 40 | .next = NULL, |
41 | .priority = 0 | 41 | .priority = NMI_LOW_PRIOR, |
42 | }; | 42 | }; |
43 | 43 | ||
44 | static int timer_start(void) | 44 | static int timer_start(void) |
@@ -58,9 +58,6 @@ static void timer_stop(void) | |||
58 | 58 | ||
59 | int __init op_nmi_timer_init(struct oprofile_operations *ops) | 59 | int __init op_nmi_timer_init(struct oprofile_operations *ops) |
60 | { | 60 | { |
61 | if ((nmi_watchdog != NMI_IO_APIC) || (atomic_read(&nmi_active) <= 0)) | ||
62 | return -ENODEV; | ||
63 | |||
64 | ops->start = timer_start; | 61 | ops->start = timer_start; |
65 | ops->stop = timer_stop; | 62 | ops->stop = timer_stop; |
66 | ops->cpu_type = "timer"; | 63 | ops->cpu_type = "timer"; |
diff --git a/arch/x86/oprofile/op_counter.h b/arch/x86/oprofile/op_counter.h index e28398df0df2..0b7b7b179cbe 100644 --- a/arch/x86/oprofile/op_counter.h +++ b/arch/x86/oprofile/op_counter.h | |||
@@ -22,6 +22,7 @@ struct op_counter_config { | |||
22 | unsigned long kernel; | 22 | unsigned long kernel; |
23 | unsigned long user; | 23 | unsigned long user; |
24 | unsigned long unit_mask; | 24 | unsigned long unit_mask; |
25 | unsigned long extra; | ||
25 | }; | 26 | }; |
26 | 27 | ||
27 | extern struct op_counter_config counter_config[]; | 28 | extern struct op_counter_config counter_config[]; |
diff --git a/arch/x86/oprofile/op_model_amd.c b/arch/x86/oprofile/op_model_amd.c index b67a6b5aa8d4..9cbb710dc94b 100644 --- a/arch/x86/oprofile/op_model_amd.c +++ b/arch/x86/oprofile/op_model_amd.c | |||
@@ -29,11 +29,12 @@ | |||
29 | #include "op_x86_model.h" | 29 | #include "op_x86_model.h" |
30 | #include "op_counter.h" | 30 | #include "op_counter.h" |
31 | 31 | ||
32 | #define NUM_COUNTERS 4 | 32 | #define NUM_COUNTERS 4 |
33 | #define NUM_COUNTERS_F15H 6 | ||
33 | #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX | 34 | #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX |
34 | #define NUM_VIRT_COUNTERS 32 | 35 | #define NUM_VIRT_COUNTERS 32 |
35 | #else | 36 | #else |
36 | #define NUM_VIRT_COUNTERS NUM_COUNTERS | 37 | #define NUM_VIRT_COUNTERS 0 |
37 | #endif | 38 | #endif |
38 | 39 | ||
39 | #define OP_EVENT_MASK 0x0FFF | 40 | #define OP_EVENT_MASK 0x0FFF |
@@ -41,38 +42,61 @@ | |||
41 | 42 | ||
42 | #define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)|(1ULL<<21)) | 43 | #define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)|(1ULL<<21)) |
43 | 44 | ||
44 | static unsigned long reset_value[NUM_VIRT_COUNTERS]; | 45 | static int num_counters; |
46 | static unsigned long reset_value[OP_MAX_COUNTER]; | ||
45 | 47 | ||
46 | #define IBS_FETCH_SIZE 6 | 48 | #define IBS_FETCH_SIZE 6 |
47 | #define IBS_OP_SIZE 12 | 49 | #define IBS_OP_SIZE 12 |
48 | 50 | ||
49 | static u32 ibs_caps; | 51 | static u32 ibs_caps; |
50 | 52 | ||
51 | struct op_ibs_config { | 53 | struct ibs_config { |
52 | unsigned long op_enabled; | 54 | unsigned long op_enabled; |
53 | unsigned long fetch_enabled; | 55 | unsigned long fetch_enabled; |
54 | unsigned long max_cnt_fetch; | 56 | unsigned long max_cnt_fetch; |
55 | unsigned long max_cnt_op; | 57 | unsigned long max_cnt_op; |
56 | unsigned long rand_en; | 58 | unsigned long rand_en; |
57 | unsigned long dispatched_ops; | 59 | unsigned long dispatched_ops; |
60 | unsigned long branch_target; | ||
58 | }; | 61 | }; |
59 | 62 | ||
60 | static struct op_ibs_config ibs_config; | 63 | struct ibs_state { |
61 | static u64 ibs_op_ctl; | 64 | u64 ibs_op_ctl; |
65 | int branch_target; | ||
66 | unsigned long sample_size; | ||
67 | }; | ||
68 | |||
69 | static struct ibs_config ibs_config; | ||
70 | static struct ibs_state ibs_state; | ||
62 | 71 | ||
63 | /* | 72 | /* |
64 | * IBS cpuid feature detection | 73 | * IBS cpuid feature detection |
65 | */ | 74 | */ |
66 | 75 | ||
67 | #define IBS_CPUID_FEATURES 0x8000001b | 76 | #define IBS_CPUID_FEATURES 0x8000001b |
68 | 77 | ||
69 | /* | 78 | /* |
70 | * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but | 79 | * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but |
71 | * bit 0 is used to indicate the existence of IBS. | 80 | * bit 0 is used to indicate the existence of IBS. |
72 | */ | 81 | */ |
73 | #define IBS_CAPS_AVAIL (1LL<<0) | 82 | #define IBS_CAPS_AVAIL (1U<<0) |
74 | #define IBS_CAPS_RDWROPCNT (1LL<<3) | 83 | #define IBS_CAPS_FETCHSAM (1U<<1) |
75 | #define IBS_CAPS_OPCNT (1LL<<4) | 84 | #define IBS_CAPS_OPSAM (1U<<2) |
85 | #define IBS_CAPS_RDWROPCNT (1U<<3) | ||
86 | #define IBS_CAPS_OPCNT (1U<<4) | ||
87 | #define IBS_CAPS_BRNTRGT (1U<<5) | ||
88 | #define IBS_CAPS_OPCNTEXT (1U<<6) | ||
89 | |||
90 | #define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \ | ||
91 | | IBS_CAPS_FETCHSAM \ | ||
92 | | IBS_CAPS_OPSAM) | ||
93 | |||
94 | /* | ||
95 | * IBS APIC setup | ||
96 | */ | ||
97 | #define IBSCTL 0x1cc | ||
98 | #define IBSCTL_LVT_OFFSET_VALID (1ULL<<8) | ||
99 | #define IBSCTL_LVT_OFFSET_MASK 0x0F | ||
76 | 100 | ||
77 | /* | 101 | /* |
78 | * IBS randomization macros | 102 | * IBS randomization macros |
@@ -92,12 +116,12 @@ static u32 get_ibs_caps(void) | |||
92 | /* check IBS cpuid feature flags */ | 116 | /* check IBS cpuid feature flags */ |
93 | max_level = cpuid_eax(0x80000000); | 117 | max_level = cpuid_eax(0x80000000); |
94 | if (max_level < IBS_CPUID_FEATURES) | 118 | if (max_level < IBS_CPUID_FEATURES) |
95 | return IBS_CAPS_AVAIL; | 119 | return IBS_CAPS_DEFAULT; |
96 | 120 | ||
97 | ibs_caps = cpuid_eax(IBS_CPUID_FEATURES); | 121 | ibs_caps = cpuid_eax(IBS_CPUID_FEATURES); |
98 | if (!(ibs_caps & IBS_CAPS_AVAIL)) | 122 | if (!(ibs_caps & IBS_CAPS_AVAIL)) |
99 | /* cpuid flags not valid */ | 123 | /* cpuid flags not valid */ |
100 | return IBS_CAPS_AVAIL; | 124 | return IBS_CAPS_DEFAULT; |
101 | 125 | ||
102 | return ibs_caps; | 126 | return ibs_caps; |
103 | } | 127 | } |
@@ -190,8 +214,8 @@ op_amd_handle_ibs(struct pt_regs * const regs, | |||
190 | rdmsrl(MSR_AMD64_IBSOPCTL, ctl); | 214 | rdmsrl(MSR_AMD64_IBSOPCTL, ctl); |
191 | if (ctl & IBS_OP_VAL) { | 215 | if (ctl & IBS_OP_VAL) { |
192 | rdmsrl(MSR_AMD64_IBSOPRIP, val); | 216 | rdmsrl(MSR_AMD64_IBSOPRIP, val); |
193 | oprofile_write_reserve(&entry, regs, val, | 217 | oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE, |
194 | IBS_OP_CODE, IBS_OP_SIZE); | 218 | ibs_state.sample_size); |
195 | oprofile_add_data64(&entry, val); | 219 | oprofile_add_data64(&entry, val); |
196 | rdmsrl(MSR_AMD64_IBSOPDATA, val); | 220 | rdmsrl(MSR_AMD64_IBSOPDATA, val); |
197 | oprofile_add_data64(&entry, val); | 221 | oprofile_add_data64(&entry, val); |
@@ -203,10 +227,14 @@ op_amd_handle_ibs(struct pt_regs * const regs, | |||
203 | oprofile_add_data64(&entry, val); | 227 | oprofile_add_data64(&entry, val); |
204 | rdmsrl(MSR_AMD64_IBSDCPHYSAD, val); | 228 | rdmsrl(MSR_AMD64_IBSDCPHYSAD, val); |
205 | oprofile_add_data64(&entry, val); | 229 | oprofile_add_data64(&entry, val); |
230 | if (ibs_state.branch_target) { | ||
231 | rdmsrl(MSR_AMD64_IBSBRTARGET, val); | ||
232 | oprofile_add_data(&entry, (unsigned long)val); | ||
233 | } | ||
206 | oprofile_write_commit(&entry); | 234 | oprofile_write_commit(&entry); |
207 | 235 | ||
208 | /* reenable the IRQ */ | 236 | /* reenable the IRQ */ |
209 | ctl = op_amd_randomize_ibs_op(ibs_op_ctl); | 237 | ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl); |
210 | wrmsrl(MSR_AMD64_IBSOPCTL, ctl); | 238 | wrmsrl(MSR_AMD64_IBSOPCTL, ctl); |
211 | } | 239 | } |
212 | } | 240 | } |
@@ -219,21 +247,32 @@ static inline void op_amd_start_ibs(void) | |||
219 | if (!ibs_caps) | 247 | if (!ibs_caps) |
220 | return; | 248 | return; |
221 | 249 | ||
250 | memset(&ibs_state, 0, sizeof(ibs_state)); | ||
251 | |||
252 | /* | ||
253 | * Note: Since the max count settings may out of range we | ||
254 | * write back the actual used values so that userland can read | ||
255 | * it. | ||
256 | */ | ||
257 | |||
222 | if (ibs_config.fetch_enabled) { | 258 | if (ibs_config.fetch_enabled) { |
223 | val = (ibs_config.max_cnt_fetch >> 4) & IBS_FETCH_MAX_CNT; | 259 | val = ibs_config.max_cnt_fetch >> 4; |
260 | val = min(val, IBS_FETCH_MAX_CNT); | ||
261 | ibs_config.max_cnt_fetch = val << 4; | ||
224 | val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0; | 262 | val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0; |
225 | val |= IBS_FETCH_ENABLE; | 263 | val |= IBS_FETCH_ENABLE; |
226 | wrmsrl(MSR_AMD64_IBSFETCHCTL, val); | 264 | wrmsrl(MSR_AMD64_IBSFETCHCTL, val); |
227 | } | 265 | } |
228 | 266 | ||
229 | if (ibs_config.op_enabled) { | 267 | if (ibs_config.op_enabled) { |
230 | ibs_op_ctl = ibs_config.max_cnt_op >> 4; | 268 | val = ibs_config.max_cnt_op >> 4; |
231 | if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) { | 269 | if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) { |
232 | /* | 270 | /* |
233 | * IbsOpCurCnt not supported. See | 271 | * IbsOpCurCnt not supported. See |
234 | * op_amd_randomize_ibs_op() for details. | 272 | * op_amd_randomize_ibs_op() for details. |
235 | */ | 273 | */ |
236 | ibs_op_ctl = clamp(ibs_op_ctl, 0x0081ULL, 0xFF80ULL); | 274 | val = clamp(val, 0x0081ULL, 0xFF80ULL); |
275 | ibs_config.max_cnt_op = val << 4; | ||
237 | } else { | 276 | } else { |
238 | /* | 277 | /* |
239 | * The start value is randomized with a | 278 | * The start value is randomized with a |
@@ -241,13 +280,24 @@ static inline void op_amd_start_ibs(void) | |||
241 | * with the half of the randomized range. Also | 280 | * with the half of the randomized range. Also |
242 | * avoid underflows. | 281 | * avoid underflows. |
243 | */ | 282 | */ |
244 | ibs_op_ctl = min(ibs_op_ctl + IBS_RANDOM_MAXCNT_OFFSET, | 283 | val += IBS_RANDOM_MAXCNT_OFFSET; |
245 | IBS_OP_MAX_CNT); | 284 | if (ibs_caps & IBS_CAPS_OPCNTEXT) |
285 | val = min(val, IBS_OP_MAX_CNT_EXT); | ||
286 | else | ||
287 | val = min(val, IBS_OP_MAX_CNT); | ||
288 | ibs_config.max_cnt_op = | ||
289 | (val - IBS_RANDOM_MAXCNT_OFFSET) << 4; | ||
246 | } | 290 | } |
247 | if (ibs_caps & IBS_CAPS_OPCNT && ibs_config.dispatched_ops) | 291 | val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT); |
248 | ibs_op_ctl |= IBS_OP_CNT_CTL; | 292 | val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0; |
249 | ibs_op_ctl |= IBS_OP_ENABLE; | 293 | val |= IBS_OP_ENABLE; |
250 | val = op_amd_randomize_ibs_op(ibs_op_ctl); | 294 | ibs_state.ibs_op_ctl = val; |
295 | ibs_state.sample_size = IBS_OP_SIZE; | ||
296 | if (ibs_config.branch_target) { | ||
297 | ibs_state.branch_target = 1; | ||
298 | ibs_state.sample_size++; | ||
299 | } | ||
300 | val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl); | ||
251 | wrmsrl(MSR_AMD64_IBSOPCTL, val); | 301 | wrmsrl(MSR_AMD64_IBSOPCTL, val); |
252 | } | 302 | } |
253 | } | 303 | } |
@@ -266,6 +316,81 @@ static void op_amd_stop_ibs(void) | |||
266 | wrmsrl(MSR_AMD64_IBSOPCTL, 0); | 316 | wrmsrl(MSR_AMD64_IBSOPCTL, 0); |
267 | } | 317 | } |
268 | 318 | ||
319 | static inline int get_eilvt(int offset) | ||
320 | { | ||
321 | return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1); | ||
322 | } | ||
323 | |||
324 | static inline int put_eilvt(int offset) | ||
325 | { | ||
326 | return !setup_APIC_eilvt(offset, 0, 0, 1); | ||
327 | } | ||
328 | |||
329 | static inline int ibs_eilvt_valid(void) | ||
330 | { | ||
331 | int offset; | ||
332 | u64 val; | ||
333 | int valid = 0; | ||
334 | |||
335 | preempt_disable(); | ||
336 | |||
337 | rdmsrl(MSR_AMD64_IBSCTL, val); | ||
338 | offset = val & IBSCTL_LVT_OFFSET_MASK; | ||
339 | |||
340 | if (!(val & IBSCTL_LVT_OFFSET_VALID)) { | ||
341 | pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n", | ||
342 | smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); | ||
343 | goto out; | ||
344 | } | ||
345 | |||
346 | if (!get_eilvt(offset)) { | ||
347 | pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n", | ||
348 | smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); | ||
349 | goto out; | ||
350 | } | ||
351 | |||
352 | valid = 1; | ||
353 | out: | ||
354 | preempt_enable(); | ||
355 | |||
356 | return valid; | ||
357 | } | ||
358 | |||
359 | static inline int get_ibs_offset(void) | ||
360 | { | ||
361 | u64 val; | ||
362 | |||
363 | rdmsrl(MSR_AMD64_IBSCTL, val); | ||
364 | if (!(val & IBSCTL_LVT_OFFSET_VALID)) | ||
365 | return -EINVAL; | ||
366 | |||
367 | return val & IBSCTL_LVT_OFFSET_MASK; | ||
368 | } | ||
369 | |||
370 | static void setup_APIC_ibs(void) | ||
371 | { | ||
372 | int offset; | ||
373 | |||
374 | offset = get_ibs_offset(); | ||
375 | if (offset < 0) | ||
376 | goto failed; | ||
377 | |||
378 | if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0)) | ||
379 | return; | ||
380 | failed: | ||
381 | pr_warn("oprofile: IBS APIC setup failed on cpu #%d\n", | ||
382 | smp_processor_id()); | ||
383 | } | ||
384 | |||
385 | static void clear_APIC_ibs(void) | ||
386 | { | ||
387 | int offset; | ||
388 | |||
389 | offset = get_ibs_offset(); | ||
390 | if (offset >= 0) | ||
391 | setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1); | ||
392 | } | ||
393 | |||
269 | #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX | 394 | #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX |
270 | 395 | ||
271 | static void op_mux_switch_ctrl(struct op_x86_model_spec const *model, | 396 | static void op_mux_switch_ctrl(struct op_x86_model_spec const *model, |
@@ -275,7 +400,7 @@ static void op_mux_switch_ctrl(struct op_x86_model_spec const *model, | |||
275 | int i; | 400 | int i; |
276 | 401 | ||
277 | /* enable active counters */ | 402 | /* enable active counters */ |
278 | for (i = 0; i < NUM_COUNTERS; ++i) { | 403 | for (i = 0; i < num_counters; ++i) { |
279 | int virt = op_x86_phys_to_virt(i); | 404 | int virt = op_x86_phys_to_virt(i); |
280 | if (!reset_value[virt]) | 405 | if (!reset_value[virt]) |
281 | continue; | 406 | continue; |
@@ -294,7 +419,7 @@ static void op_amd_shutdown(struct op_msrs const * const msrs) | |||
294 | { | 419 | { |
295 | int i; | 420 | int i; |
296 | 421 | ||
297 | for (i = 0; i < NUM_COUNTERS; ++i) { | 422 | for (i = 0; i < num_counters; ++i) { |
298 | if (!msrs->counters[i].addr) | 423 | if (!msrs->counters[i].addr) |
299 | continue; | 424 | continue; |
300 | release_perfctr_nmi(MSR_K7_PERFCTR0 + i); | 425 | release_perfctr_nmi(MSR_K7_PERFCTR0 + i); |
@@ -306,7 +431,7 @@ static int op_amd_fill_in_addresses(struct op_msrs * const msrs) | |||
306 | { | 431 | { |
307 | int i; | 432 | int i; |
308 | 433 | ||
309 | for (i = 0; i < NUM_COUNTERS; i++) { | 434 | for (i = 0; i < num_counters; i++) { |
310 | if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i)) | 435 | if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i)) |
311 | goto fail; | 436 | goto fail; |
312 | if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) { | 437 | if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) { |
@@ -314,8 +439,13 @@ static int op_amd_fill_in_addresses(struct op_msrs * const msrs) | |||
314 | goto fail; | 439 | goto fail; |
315 | } | 440 | } |
316 | /* both registers must be reserved */ | 441 | /* both registers must be reserved */ |
317 | msrs->counters[i].addr = MSR_K7_PERFCTR0 + i; | 442 | if (num_counters == NUM_COUNTERS_F15H) { |
318 | msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i; | 443 | msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1); |
444 | msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1); | ||
445 | } else { | ||
446 | msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i; | ||
447 | msrs->counters[i].addr = MSR_K7_PERFCTR0 + i; | ||
448 | } | ||
319 | continue; | 449 | continue; |
320 | fail: | 450 | fail: |
321 | if (!counter_config[i].enabled) | 451 | if (!counter_config[i].enabled) |
@@ -335,7 +465,7 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, | |||
335 | int i; | 465 | int i; |
336 | 466 | ||
337 | /* setup reset_value */ | 467 | /* setup reset_value */ |
338 | for (i = 0; i < NUM_VIRT_COUNTERS; ++i) { | 468 | for (i = 0; i < OP_MAX_COUNTER; ++i) { |
339 | if (counter_config[i].enabled | 469 | if (counter_config[i].enabled |
340 | && msrs->counters[op_x86_virt_to_phys(i)].addr) | 470 | && msrs->counters[op_x86_virt_to_phys(i)].addr) |
341 | reset_value[i] = counter_config[i].count; | 471 | reset_value[i] = counter_config[i].count; |
@@ -344,7 +474,7 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, | |||
344 | } | 474 | } |
345 | 475 | ||
346 | /* clear all counters */ | 476 | /* clear all counters */ |
347 | for (i = 0; i < NUM_COUNTERS; ++i) { | 477 | for (i = 0; i < num_counters; ++i) { |
348 | if (!msrs->controls[i].addr) | 478 | if (!msrs->controls[i].addr) |
349 | continue; | 479 | continue; |
350 | rdmsrl(msrs->controls[i].addr, val); | 480 | rdmsrl(msrs->controls[i].addr, val); |
@@ -360,7 +490,7 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, | |||
360 | } | 490 | } |
361 | 491 | ||
362 | /* enable active counters */ | 492 | /* enable active counters */ |
363 | for (i = 0; i < NUM_COUNTERS; ++i) { | 493 | for (i = 0; i < num_counters; ++i) { |
364 | int virt = op_x86_phys_to_virt(i); | 494 | int virt = op_x86_phys_to_virt(i); |
365 | if (!reset_value[virt]) | 495 | if (!reset_value[virt]) |
366 | continue; | 496 | continue; |
@@ -376,13 +506,13 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, | |||
376 | } | 506 | } |
377 | 507 | ||
378 | if (ibs_caps) | 508 | if (ibs_caps) |
379 | setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0); | 509 | setup_APIC_ibs(); |
380 | } | 510 | } |
381 | 511 | ||
382 | static void op_amd_cpu_shutdown(void) | 512 | static void op_amd_cpu_shutdown(void) |
383 | { | 513 | { |
384 | if (ibs_caps) | 514 | if (ibs_caps) |
385 | setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1); | 515 | clear_APIC_ibs(); |
386 | } | 516 | } |
387 | 517 | ||
388 | static int op_amd_check_ctrs(struct pt_regs * const regs, | 518 | static int op_amd_check_ctrs(struct pt_regs * const regs, |
@@ -391,7 +521,7 @@ static int op_amd_check_ctrs(struct pt_regs * const regs, | |||
391 | u64 val; | 521 | u64 val; |
392 | int i; | 522 | int i; |
393 | 523 | ||
394 | for (i = 0; i < NUM_COUNTERS; ++i) { | 524 | for (i = 0; i < num_counters; ++i) { |
395 | int virt = op_x86_phys_to_virt(i); | 525 | int virt = op_x86_phys_to_virt(i); |
396 | if (!reset_value[virt]) | 526 | if (!reset_value[virt]) |
397 | continue; | 527 | continue; |
@@ -414,7 +544,7 @@ static void op_amd_start(struct op_msrs const * const msrs) | |||
414 | u64 val; | 544 | u64 val; |
415 | int i; | 545 | int i; |
416 | 546 | ||
417 | for (i = 0; i < NUM_COUNTERS; ++i) { | 547 | for (i = 0; i < num_counters; ++i) { |
418 | if (!reset_value[op_x86_phys_to_virt(i)]) | 548 | if (!reset_value[op_x86_phys_to_virt(i)]) |
419 | continue; | 549 | continue; |
420 | rdmsrl(msrs->controls[i].addr, val); | 550 | rdmsrl(msrs->controls[i].addr, val); |
@@ -434,7 +564,7 @@ static void op_amd_stop(struct op_msrs const * const msrs) | |||
434 | * Subtle: stop on all counters to avoid race with setting our | 564 | * Subtle: stop on all counters to avoid race with setting our |
435 | * pm callback | 565 | * pm callback |
436 | */ | 566 | */ |
437 | for (i = 0; i < NUM_COUNTERS; ++i) { | 567 | for (i = 0; i < num_counters; ++i) { |
438 | if (!reset_value[op_x86_phys_to_virt(i)]) | 568 | if (!reset_value[op_x86_phys_to_virt(i)]) |
439 | continue; | 569 | continue; |
440 | rdmsrl(msrs->controls[i].addr, val); | 570 | rdmsrl(msrs->controls[i].addr, val); |
@@ -445,16 +575,11 @@ static void op_amd_stop(struct op_msrs const * const msrs) | |||
445 | op_amd_stop_ibs(); | 575 | op_amd_stop_ibs(); |
446 | } | 576 | } |
447 | 577 | ||
448 | static int __init_ibs_nmi(void) | 578 | static int setup_ibs_ctl(int ibs_eilvt_off) |
449 | { | 579 | { |
450 | #define IBSCTL_LVTOFFSETVAL (1 << 8) | ||
451 | #define IBSCTL 0x1cc | ||
452 | struct pci_dev *cpu_cfg; | 580 | struct pci_dev *cpu_cfg; |
453 | int nodes; | 581 | int nodes; |
454 | u32 value = 0; | 582 | u32 value = 0; |
455 | u8 ibs_eilvt_off; | ||
456 | |||
457 | ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1); | ||
458 | 583 | ||
459 | nodes = 0; | 584 | nodes = 0; |
460 | cpu_cfg = NULL; | 585 | cpu_cfg = NULL; |
@@ -466,25 +591,75 @@ static int __init_ibs_nmi(void) | |||
466 | break; | 591 | break; |
467 | ++nodes; | 592 | ++nodes; |
468 | pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off | 593 | pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off |
469 | | IBSCTL_LVTOFFSETVAL); | 594 | | IBSCTL_LVT_OFFSET_VALID); |
470 | pci_read_config_dword(cpu_cfg, IBSCTL, &value); | 595 | pci_read_config_dword(cpu_cfg, IBSCTL, &value); |
471 | if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) { | 596 | if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) { |
472 | pci_dev_put(cpu_cfg); | 597 | pci_dev_put(cpu_cfg); |
473 | printk(KERN_DEBUG "Failed to setup IBS LVT offset, " | 598 | printk(KERN_DEBUG "Failed to setup IBS LVT offset, " |
474 | "IBSCTL = 0x%08x", value); | 599 | "IBSCTL = 0x%08x\n", value); |
475 | return 1; | 600 | return -EINVAL; |
476 | } | 601 | } |
477 | } while (1); | 602 | } while (1); |
478 | 603 | ||
479 | if (!nodes) { | 604 | if (!nodes) { |
480 | printk(KERN_DEBUG "No CPU node configured for IBS"); | 605 | printk(KERN_DEBUG "No CPU node configured for IBS\n"); |
481 | return 1; | 606 | return -ENODEV; |
607 | } | ||
608 | |||
609 | return 0; | ||
610 | } | ||
611 | |||
612 | /* | ||
613 | * This runs only on the current cpu. We try to find an LVT offset and | ||
614 | * setup the local APIC. For this we must disable preemption. On | ||
615 | * success we initialize all nodes with this offset. This updates then | ||
616 | * the offset in the IBS_CTL per-node msr. The per-core APIC setup of | ||
617 | * the IBS interrupt vector is called from op_amd_setup_ctrs()/op_- | ||
618 | * amd_cpu_shutdown() using the new offset. | ||
619 | */ | ||
620 | static int force_ibs_eilvt_setup(void) | ||
621 | { | ||
622 | int offset; | ||
623 | int ret; | ||
624 | |||
625 | preempt_disable(); | ||
626 | /* find the next free available EILVT entry, skip offset 0 */ | ||
627 | for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) { | ||
628 | if (get_eilvt(offset)) | ||
629 | break; | ||
630 | } | ||
631 | preempt_enable(); | ||
632 | |||
633 | if (offset == APIC_EILVT_NR_MAX) { | ||
634 | printk(KERN_DEBUG "No EILVT entry available\n"); | ||
635 | return -EBUSY; | ||
482 | } | 636 | } |
483 | 637 | ||
638 | ret = setup_ibs_ctl(offset); | ||
639 | if (ret) | ||
640 | goto out; | ||
641 | |||
642 | if (!ibs_eilvt_valid()) { | ||
643 | ret = -EFAULT; | ||
644 | goto out; | ||
645 | } | ||
646 | |||
647 | pr_err(FW_BUG "using offset %d for IBS interrupts\n", offset); | ||
648 | pr_err(FW_BUG "workaround enabled for IBS LVT offset\n"); | ||
649 | |||
484 | return 0; | 650 | return 0; |
651 | out: | ||
652 | preempt_disable(); | ||
653 | put_eilvt(offset); | ||
654 | preempt_enable(); | ||
655 | return ret; | ||
485 | } | 656 | } |
486 | 657 | ||
487 | /* initialize the APIC for the IBS interrupts if available */ | 658 | /* |
659 | * check and reserve APIC extended interrupt LVT offset for IBS if | ||
660 | * available | ||
661 | */ | ||
662 | |||
488 | static void init_ibs(void) | 663 | static void init_ibs(void) |
489 | { | 664 | { |
490 | ibs_caps = get_ibs_caps(); | 665 | ibs_caps = get_ibs_caps(); |
@@ -492,13 +667,18 @@ static void init_ibs(void) | |||
492 | if (!ibs_caps) | 667 | if (!ibs_caps) |
493 | return; | 668 | return; |
494 | 669 | ||
495 | if (__init_ibs_nmi()) { | 670 | if (ibs_eilvt_valid()) |
496 | ibs_caps = 0; | 671 | goto out; |
497 | return; | 672 | |
498 | } | 673 | if (!force_ibs_eilvt_setup()) |
674 | goto out; | ||
499 | 675 | ||
500 | printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", | 676 | /* Failed to setup ibs */ |
501 | (unsigned)ibs_caps); | 677 | ibs_caps = 0; |
678 | return; | ||
679 | |||
680 | out: | ||
681 | printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps); | ||
502 | } | 682 | } |
503 | 683 | ||
504 | static int (*create_arch_files)(struct super_block *sb, struct dentry *root); | 684 | static int (*create_arch_files)(struct super_block *sb, struct dentry *root); |
@@ -521,44 +701,60 @@ static int setup_ibs_files(struct super_block *sb, struct dentry *root) | |||
521 | /* model specific files */ | 701 | /* model specific files */ |
522 | 702 | ||
523 | /* setup some reasonable defaults */ | 703 | /* setup some reasonable defaults */ |
704 | memset(&ibs_config, 0, sizeof(ibs_config)); | ||
524 | ibs_config.max_cnt_fetch = 250000; | 705 | ibs_config.max_cnt_fetch = 250000; |
525 | ibs_config.fetch_enabled = 0; | ||
526 | ibs_config.max_cnt_op = 250000; | 706 | ibs_config.max_cnt_op = 250000; |
527 | ibs_config.op_enabled = 0; | 707 | |
528 | ibs_config.dispatched_ops = 0; | 708 | if (ibs_caps & IBS_CAPS_FETCHSAM) { |
529 | 709 | dir = oprofilefs_mkdir(sb, root, "ibs_fetch"); | |
530 | dir = oprofilefs_mkdir(sb, root, "ibs_fetch"); | 710 | oprofilefs_create_ulong(sb, dir, "enable", |
531 | oprofilefs_create_ulong(sb, dir, "enable", | 711 | &ibs_config.fetch_enabled); |
532 | &ibs_config.fetch_enabled); | 712 | oprofilefs_create_ulong(sb, dir, "max_count", |
533 | oprofilefs_create_ulong(sb, dir, "max_count", | 713 | &ibs_config.max_cnt_fetch); |
534 | &ibs_config.max_cnt_fetch); | 714 | oprofilefs_create_ulong(sb, dir, "rand_enable", |
535 | oprofilefs_create_ulong(sb, dir, "rand_enable", | 715 | &ibs_config.rand_en); |
536 | &ibs_config.rand_en); | 716 | } |
537 | 717 | ||
538 | dir = oprofilefs_mkdir(sb, root, "ibs_op"); | 718 | if (ibs_caps & IBS_CAPS_OPSAM) { |
539 | oprofilefs_create_ulong(sb, dir, "enable", | 719 | dir = oprofilefs_mkdir(sb, root, "ibs_op"); |
540 | &ibs_config.op_enabled); | 720 | oprofilefs_create_ulong(sb, dir, "enable", |
541 | oprofilefs_create_ulong(sb, dir, "max_count", | 721 | &ibs_config.op_enabled); |
542 | &ibs_config.max_cnt_op); | 722 | oprofilefs_create_ulong(sb, dir, "max_count", |
543 | if (ibs_caps & IBS_CAPS_OPCNT) | 723 | &ibs_config.max_cnt_op); |
544 | oprofilefs_create_ulong(sb, dir, "dispatched_ops", | 724 | if (ibs_caps & IBS_CAPS_OPCNT) |
545 | &ibs_config.dispatched_ops); | 725 | oprofilefs_create_ulong(sb, dir, "dispatched_ops", |
726 | &ibs_config.dispatched_ops); | ||
727 | if (ibs_caps & IBS_CAPS_BRNTRGT) | ||
728 | oprofilefs_create_ulong(sb, dir, "branch_target", | ||
729 | &ibs_config.branch_target); | ||
730 | } | ||
546 | 731 | ||
547 | return 0; | 732 | return 0; |
548 | } | 733 | } |
549 | 734 | ||
735 | struct op_x86_model_spec op_amd_spec; | ||
736 | |||
550 | static int op_amd_init(struct oprofile_operations *ops) | 737 | static int op_amd_init(struct oprofile_operations *ops) |
551 | { | 738 | { |
552 | init_ibs(); | 739 | init_ibs(); |
553 | create_arch_files = ops->create_files; | 740 | create_arch_files = ops->create_files; |
554 | ops->create_files = setup_ibs_files; | 741 | ops->create_files = setup_ibs_files; |
742 | |||
743 | if (boot_cpu_data.x86 == 0x15) { | ||
744 | num_counters = NUM_COUNTERS_F15H; | ||
745 | } else { | ||
746 | num_counters = NUM_COUNTERS; | ||
747 | } | ||
748 | |||
749 | op_amd_spec.num_counters = num_counters; | ||
750 | op_amd_spec.num_controls = num_counters; | ||
751 | op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS); | ||
752 | |||
555 | return 0; | 753 | return 0; |
556 | } | 754 | } |
557 | 755 | ||
558 | struct op_x86_model_spec op_amd_spec = { | 756 | struct op_x86_model_spec op_amd_spec = { |
559 | .num_counters = NUM_COUNTERS, | 757 | /* num_counters/num_controls filled in at runtime */ |
560 | .num_controls = NUM_COUNTERS, | ||
561 | .num_virt_counters = NUM_VIRT_COUNTERS, | ||
562 | .reserved = MSR_AMD_EVENTSEL_RESERVED, | 758 | .reserved = MSR_AMD_EVENTSEL_RESERVED, |
563 | .event_mask = OP_EVENT_MASK, | 759 | .event_mask = OP_EVENT_MASK, |
564 | .init = op_amd_init, | 760 | .init = op_amd_init, |
diff --git a/arch/x86/oprofile/op_model_p4.c b/arch/x86/oprofile/op_model_p4.c index 182558dd5515..98ab13058f89 100644 --- a/arch/x86/oprofile/op_model_p4.c +++ b/arch/x86/oprofile/op_model_p4.c | |||
@@ -11,7 +11,7 @@ | |||
11 | #include <linux/oprofile.h> | 11 | #include <linux/oprofile.h> |
12 | #include <linux/smp.h> | 12 | #include <linux/smp.h> |
13 | #include <linux/ptrace.h> | 13 | #include <linux/ptrace.h> |
14 | #include <linux/nmi.h> | 14 | #include <asm/nmi.h> |
15 | #include <asm/msr.h> | 15 | #include <asm/msr.h> |
16 | #include <asm/fixmap.h> | 16 | #include <asm/fixmap.h> |
17 | #include <asm/apic.h> | 17 | #include <asm/apic.h> |
@@ -50,7 +50,7 @@ static inline void setup_num_counters(void) | |||
50 | #endif | 50 | #endif |
51 | } | 51 | } |
52 | 52 | ||
53 | static int inline addr_increment(void) | 53 | static inline int addr_increment(void) |
54 | { | 54 | { |
55 | #ifdef CONFIG_SMP | 55 | #ifdef CONFIG_SMP |
56 | return smp_num_siblings == 2 ? 2 : 1; | 56 | return smp_num_siblings == 2 ? 2 : 1; |
diff --git a/arch/x86/oprofile/op_model_ppro.c b/arch/x86/oprofile/op_model_ppro.c index d769cda54082..94b745045e45 100644 --- a/arch/x86/oprofile/op_model_ppro.c +++ b/arch/x86/oprofile/op_model_ppro.c | |||
@@ -95,8 +95,8 @@ static void ppro_setup_ctrs(struct op_x86_model_spec const *model, | |||
95 | * counter width: | 95 | * counter width: |
96 | */ | 96 | */ |
97 | if (!(eax.split.version_id == 0 && | 97 | if (!(eax.split.version_id == 0 && |
98 | current_cpu_data.x86 == 6 && | 98 | __this_cpu_read(cpu_info.x86) == 6 && |
99 | current_cpu_data.x86_model == 15)) { | 99 | __this_cpu_read(cpu_info.x86_model) == 15)) { |
100 | 100 | ||
101 | if (counter_width < eax.split.bit_width) | 101 | if (counter_width < eax.split.bit_width) |
102 | counter_width = eax.split.bit_width; | 102 | counter_width = eax.split.bit_width; |
@@ -235,8 +235,8 @@ static void arch_perfmon_setup_counters(void) | |||
235 | eax.full = cpuid_eax(0xa); | 235 | eax.full = cpuid_eax(0xa); |
236 | 236 | ||
237 | /* Workaround for BIOS bugs in 6/15. Taken from perfmon2 */ | 237 | /* Workaround for BIOS bugs in 6/15. Taken from perfmon2 */ |
238 | if (eax.split.version_id == 0 && current_cpu_data.x86 == 6 && | 238 | if (eax.split.version_id == 0 && __this_cpu_read(cpu_info.x86) == 6 && |
239 | current_cpu_data.x86_model == 15) { | 239 | __this_cpu_read(cpu_info.x86_model) == 15) { |
240 | eax.split.version_id = 2; | 240 | eax.split.version_id = 2; |
241 | eax.split.num_counters = 2; | 241 | eax.split.num_counters = 2; |
242 | eax.split.bit_width = 40; | 242 | eax.split.bit_width = 40; |