diff options
-rw-r--r-- | arch/x86/include/asm/desc.h | 72 | ||||
-rw-r--r-- | arch/x86/include/asm/entry_arch.h | 8 | ||||
-rw-r--r-- | arch/x86/include/asm/hw_irq.h | 17 | ||||
-rw-r--r-- | arch/x86/include/asm/mshyperv.h | 3 | ||||
-rw-r--r-- | arch/x86/include/asm/trace/irq_vectors.h | 104 | ||||
-rw-r--r-- | arch/x86/include/asm/uv/uv_bau.h | 3 | ||||
-rw-r--r-- | arch/x86/kernel/Makefile | 1 | ||||
-rw-r--r-- | arch/x86/kernel/apic/Makefile | 1 | ||||
-rw-r--r-- | arch/x86/kernel/apic/apic.c | 42 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/common.c | 4 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/mcheck/therm_throt.c | 10 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/mcheck/threshold.c | 10 | ||||
-rw-r--r-- | arch/x86/kernel/entry_32.S | 12 | ||||
-rw-r--r-- | arch/x86/kernel/entry_64.S | 31 | ||||
-rw-r--r-- | arch/x86/kernel/head_64.S | 6 | ||||
-rw-r--r-- | arch/x86/kernel/irq.c | 13 | ||||
-rw-r--r-- | arch/x86/kernel/irq_work.c | 10 | ||||
-rw-r--r-- | arch/x86/kernel/smp.c | 30 | ||||
-rw-r--r-- | arch/x86/kernel/tracepoint.c | 57 | ||||
-rw-r--r-- | include/xen/events.h | 3 |
20 files changed, 422 insertions, 15 deletions
diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h index af290b8f124a..1377ecb29d8d 100644 --- a/arch/x86/include/asm/desc.h +++ b/arch/x86/include/asm/desc.h | |||
@@ -320,6 +320,19 @@ static inline void set_nmi_gate(int gate, void *addr) | |||
320 | } | 320 | } |
321 | #endif | 321 | #endif |
322 | 322 | ||
323 | #ifdef CONFIG_TRACING | ||
324 | extern struct desc_ptr trace_idt_descr; | ||
325 | extern gate_desc trace_idt_table[]; | ||
326 | static inline void write_trace_idt_entry(int entry, const gate_desc *gate) | ||
327 | { | ||
328 | write_idt_entry(trace_idt_table, entry, gate); | ||
329 | } | ||
330 | #else | ||
331 | static inline void write_trace_idt_entry(int entry, const gate_desc *gate) | ||
332 | { | ||
333 | } | ||
334 | #endif | ||
335 | |||
323 | static inline void _set_gate(int gate, unsigned type, void *addr, | 336 | static inline void _set_gate(int gate, unsigned type, void *addr, |
324 | unsigned dpl, unsigned ist, unsigned seg) | 337 | unsigned dpl, unsigned ist, unsigned seg) |
325 | { | 338 | { |
@@ -331,6 +344,7 @@ static inline void _set_gate(int gate, unsigned type, void *addr, | |||
331 | * setup time | 344 | * setup time |
332 | */ | 345 | */ |
333 | write_idt_entry(idt_table, gate, &s); | 346 | write_idt_entry(idt_table, gate, &s); |
347 | write_trace_idt_entry(gate, &s); | ||
334 | } | 348 | } |
335 | 349 | ||
336 | /* | 350 | /* |
@@ -360,12 +374,39 @@ static inline void alloc_system_vector(int vector) | |||
360 | } | 374 | } |
361 | } | 375 | } |
362 | 376 | ||
363 | static inline void alloc_intr_gate(unsigned int n, void *addr) | 377 | #ifdef CONFIG_TRACING |
378 | static inline void trace_set_intr_gate(unsigned int gate, void *addr) | ||
379 | { | ||
380 | gate_desc s; | ||
381 | |||
382 | pack_gate(&s, GATE_INTERRUPT, (unsigned long)addr, 0, 0, __KERNEL_CS); | ||
383 | write_idt_entry(trace_idt_table, gate, &s); | ||
384 | } | ||
385 | |||
386 | static inline void __trace_alloc_intr_gate(unsigned int n, void *addr) | ||
387 | { | ||
388 | trace_set_intr_gate(n, addr); | ||
389 | } | ||
390 | #else | ||
391 | static inline void trace_set_intr_gate(unsigned int gate, void *addr) | ||
392 | { | ||
393 | } | ||
394 | |||
395 | #define __trace_alloc_intr_gate(n, addr) | ||
396 | #endif | ||
397 | |||
398 | static inline void __alloc_intr_gate(unsigned int n, void *addr) | ||
364 | { | 399 | { |
365 | alloc_system_vector(n); | ||
366 | set_intr_gate(n, addr); | 400 | set_intr_gate(n, addr); |
367 | } | 401 | } |
368 | 402 | ||
403 | #define alloc_intr_gate(n, addr) \ | ||
404 | do { \ | ||
405 | alloc_system_vector(n); \ | ||
406 | __alloc_intr_gate(n, addr); \ | ||
407 | __trace_alloc_intr_gate(n, trace_##addr); \ | ||
408 | } while (0) | ||
409 | |||
369 | /* | 410 | /* |
370 | * This routine sets up an interrupt gate at directory privilege level 3. | 411 | * This routine sets up an interrupt gate at directory privilege level 3. |
371 | */ | 412 | */ |
@@ -430,6 +471,31 @@ static inline void load_debug_idt(void) | |||
430 | } | 471 | } |
431 | #endif | 472 | #endif |
432 | 473 | ||
474 | #ifdef CONFIG_TRACING | ||
475 | extern atomic_t trace_idt_ctr; | ||
476 | static inline bool is_trace_idt_enabled(void) | ||
477 | { | ||
478 | if (atomic_read(&trace_idt_ctr)) | ||
479 | return true; | ||
480 | |||
481 | return false; | ||
482 | } | ||
483 | |||
484 | static inline void load_trace_idt(void) | ||
485 | { | ||
486 | load_idt((const struct desc_ptr *)&trace_idt_descr); | ||
487 | } | ||
488 | #else | ||
489 | static inline bool is_trace_idt_enabled(void) | ||
490 | { | ||
491 | return false; | ||
492 | } | ||
493 | |||
494 | static inline void load_trace_idt(void) | ||
495 | { | ||
496 | } | ||
497 | #endif | ||
498 | |||
433 | /* | 499 | /* |
434 | * the load_current_idt() is called with interrupt disabled by local_irq_save() | 500 | * the load_current_idt() is called with interrupt disabled by local_irq_save() |
435 | * to avoid races. That way the IDT will always be set back to the expected | 501 | * to avoid races. That way the IDT will always be set back to the expected |
@@ -442,6 +508,8 @@ static inline void load_current_idt(void) | |||
442 | local_irq_save(flags); | 508 | local_irq_save(flags); |
443 | if (is_debug_idt_enabled()) | 509 | if (is_debug_idt_enabled()) |
444 | load_debug_idt(); | 510 | load_debug_idt(); |
511 | else if (is_trace_idt_enabled()) | ||
512 | load_trace_idt(); | ||
445 | else | 513 | else |
446 | load_idt((const struct desc_ptr *)&idt_descr); | 514 | load_idt((const struct desc_ptr *)&idt_descr); |
447 | local_irq_restore(flags); | 515 | local_irq_restore(flags); |
diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h index 9bd4ecac72be..dc5fa661465f 100644 --- a/arch/x86/include/asm/entry_arch.h +++ b/arch/x86/include/asm/entry_arch.h | |||
@@ -13,14 +13,16 @@ | |||
13 | BUILD_INTERRUPT(reschedule_interrupt,RESCHEDULE_VECTOR) | 13 | BUILD_INTERRUPT(reschedule_interrupt,RESCHEDULE_VECTOR) |
14 | BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR) | 14 | BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR) |
15 | BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR) | 15 | BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR) |
16 | BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR) | 16 | BUILD_INTERRUPT3(irq_move_cleanup_interrupt, IRQ_MOVE_CLEANUP_VECTOR, |
17 | BUILD_INTERRUPT(reboot_interrupt,REBOOT_VECTOR) | 17 | smp_irq_move_cleanup_interrupt) |
18 | BUILD_INTERRUPT3(reboot_interrupt, REBOOT_VECTOR, smp_reboot_interrupt) | ||
18 | #endif | 19 | #endif |
19 | 20 | ||
20 | BUILD_INTERRUPT(x86_platform_ipi, X86_PLATFORM_IPI_VECTOR) | 21 | BUILD_INTERRUPT(x86_platform_ipi, X86_PLATFORM_IPI_VECTOR) |
21 | 22 | ||
22 | #ifdef CONFIG_HAVE_KVM | 23 | #ifdef CONFIG_HAVE_KVM |
23 | BUILD_INTERRUPT(kvm_posted_intr_ipi, POSTED_INTR_VECTOR) | 24 | BUILD_INTERRUPT3(kvm_posted_intr_ipi, POSTED_INTR_VECTOR, |
25 | smp_kvm_posted_intr_ipi) | ||
24 | #endif | 26 | #endif |
25 | 27 | ||
26 | /* | 28 | /* |
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index 1da97efad08a..e4ac559c4a24 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h | |||
@@ -77,6 +77,23 @@ extern void threshold_interrupt(void); | |||
77 | extern void call_function_interrupt(void); | 77 | extern void call_function_interrupt(void); |
78 | extern void call_function_single_interrupt(void); | 78 | extern void call_function_single_interrupt(void); |
79 | 79 | ||
80 | #ifdef CONFIG_TRACING | ||
81 | /* Interrupt handlers registered during init_IRQ */ | ||
82 | extern void trace_apic_timer_interrupt(void); | ||
83 | extern void trace_x86_platform_ipi(void); | ||
84 | extern void trace_error_interrupt(void); | ||
85 | extern void trace_irq_work_interrupt(void); | ||
86 | extern void trace_spurious_interrupt(void); | ||
87 | extern void trace_thermal_interrupt(void); | ||
88 | extern void trace_reschedule_interrupt(void); | ||
89 | extern void trace_threshold_interrupt(void); | ||
90 | extern void trace_call_function_interrupt(void); | ||
91 | extern void trace_call_function_single_interrupt(void); | ||
92 | #define trace_irq_move_cleanup_interrupt irq_move_cleanup_interrupt | ||
93 | #define trace_reboot_interrupt reboot_interrupt | ||
94 | #define trace_kvm_posted_intr_ipi kvm_posted_intr_ipi | ||
95 | #endif /* CONFIG_TRACING */ | ||
96 | |||
80 | /* IOAPIC */ | 97 | /* IOAPIC */ |
81 | #define IO_APIC_IRQ(x) (((x) >= NR_IRQS_LEGACY) || ((1<<(x)) & io_apic_irqs)) | 98 | #define IO_APIC_IRQ(x) (((x) >= NR_IRQS_LEGACY) || ((1<<(x)) & io_apic_irqs)) |
82 | extern unsigned long io_apic_irqs; | 99 | extern unsigned long io_apic_irqs; |
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index c2934be2446a..cd9c41938b8a 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h | |||
@@ -12,6 +12,9 @@ struct ms_hyperv_info { | |||
12 | extern struct ms_hyperv_info ms_hyperv; | 12 | extern struct ms_hyperv_info ms_hyperv; |
13 | 13 | ||
14 | void hyperv_callback_vector(void); | 14 | void hyperv_callback_vector(void); |
15 | #ifdef CONFIG_TRACING | ||
16 | #define trace_hyperv_callback_vector hyperv_callback_vector | ||
17 | #endif | ||
15 | void hyperv_vector_handler(struct pt_regs *regs); | 18 | void hyperv_vector_handler(struct pt_regs *regs); |
16 | void hv_register_vmbus_handler(int irq, irq_handler_t handler); | 19 | void hv_register_vmbus_handler(int irq, irq_handler_t handler); |
17 | 20 | ||
diff --git a/arch/x86/include/asm/trace/irq_vectors.h b/arch/x86/include/asm/trace/irq_vectors.h new file mode 100644 index 000000000000..2874df24e7a4 --- /dev/null +++ b/arch/x86/include/asm/trace/irq_vectors.h | |||
@@ -0,0 +1,104 @@ | |||
1 | #undef TRACE_SYSTEM | ||
2 | #define TRACE_SYSTEM irq_vectors | ||
3 | |||
4 | #if !defined(_TRACE_IRQ_VECTORS_H) || defined(TRACE_HEADER_MULTI_READ) | ||
5 | #define _TRACE_IRQ_VECTORS_H | ||
6 | |||
7 | #include <linux/tracepoint.h> | ||
8 | |||
9 | extern void trace_irq_vector_regfunc(void); | ||
10 | extern void trace_irq_vector_unregfunc(void); | ||
11 | |||
12 | DECLARE_EVENT_CLASS(x86_irq_vector, | ||
13 | |||
14 | TP_PROTO(int vector), | ||
15 | |||
16 | TP_ARGS(vector), | ||
17 | |||
18 | TP_STRUCT__entry( | ||
19 | __field( int, vector ) | ||
20 | ), | ||
21 | |||
22 | TP_fast_assign( | ||
23 | __entry->vector = vector; | ||
24 | ), | ||
25 | |||
26 | TP_printk("vector=%d", __entry->vector) ); | ||
27 | |||
28 | #define DEFINE_IRQ_VECTOR_EVENT(name) \ | ||
29 | DEFINE_EVENT_FN(x86_irq_vector, name##_entry, \ | ||
30 | TP_PROTO(int vector), \ | ||
31 | TP_ARGS(vector), \ | ||
32 | trace_irq_vector_regfunc, \ | ||
33 | trace_irq_vector_unregfunc); \ | ||
34 | DEFINE_EVENT_FN(x86_irq_vector, name##_exit, \ | ||
35 | TP_PROTO(int vector), \ | ||
36 | TP_ARGS(vector), \ | ||
37 | trace_irq_vector_regfunc, \ | ||
38 | trace_irq_vector_unregfunc); | ||
39 | |||
40 | |||
41 | /* | ||
42 | * local_timer - called when entering/exiting a local timer interrupt | ||
43 | * vector handler | ||
44 | */ | ||
45 | DEFINE_IRQ_VECTOR_EVENT(local_timer); | ||
46 | |||
47 | /* | ||
48 | * reschedule - called when entering/exiting a reschedule vector handler | ||
49 | */ | ||
50 | DEFINE_IRQ_VECTOR_EVENT(reschedule); | ||
51 | |||
52 | /* | ||
53 | * spurious_apic - called when entering/exiting a spurious apic vector handler | ||
54 | */ | ||
55 | DEFINE_IRQ_VECTOR_EVENT(spurious_apic); | ||
56 | |||
57 | /* | ||
58 | * error_apic - called when entering/exiting an error apic vector handler | ||
59 | */ | ||
60 | DEFINE_IRQ_VECTOR_EVENT(error_apic); | ||
61 | |||
62 | /* | ||
63 | * x86_platform_ipi - called when entering/exiting a x86 platform ipi interrupt | ||
64 | * vector handler | ||
65 | */ | ||
66 | DEFINE_IRQ_VECTOR_EVENT(x86_platform_ipi); | ||
67 | |||
68 | /* | ||
69 | * irq_work - called when entering/exiting a irq work interrupt | ||
70 | * vector handler | ||
71 | */ | ||
72 | DEFINE_IRQ_VECTOR_EVENT(irq_work); | ||
73 | |||
74 | /* | ||
75 | * call_function - called when entering/exiting a call function interrupt | ||
76 | * vector handler | ||
77 | */ | ||
78 | DEFINE_IRQ_VECTOR_EVENT(call_function); | ||
79 | |||
80 | /* | ||
81 | * call_function_single - called when entering/exiting a call function | ||
82 | * single interrupt vector handler | ||
83 | */ | ||
84 | DEFINE_IRQ_VECTOR_EVENT(call_function_single); | ||
85 | |||
86 | /* | ||
87 | * threshold_apic - called when entering/exiting a threshold apic interrupt | ||
88 | * vector handler | ||
89 | */ | ||
90 | DEFINE_IRQ_VECTOR_EVENT(threshold_apic); | ||
91 | |||
92 | /* | ||
93 | * thermal_apic - called when entering/exiting a thermal apic interrupt | ||
94 | * vector handler | ||
95 | */ | ||
96 | DEFINE_IRQ_VECTOR_EVENT(thermal_apic); | ||
97 | |||
98 | #undef TRACE_INCLUDE_PATH | ||
99 | #define TRACE_INCLUDE_PATH . | ||
100 | #define TRACE_INCLUDE_FILE irq_vectors | ||
101 | #endif /* _TRACE_IRQ_VECTORS_H */ | ||
102 | |||
103 | /* This part must be outside protection */ | ||
104 | #include <trace/define_trace.h> | ||
diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h index a06983cdc125..0b46ef261c77 100644 --- a/arch/x86/include/asm/uv/uv_bau.h +++ b/arch/x86/include/asm/uv/uv_bau.h | |||
@@ -731,6 +731,9 @@ static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits) | |||
731 | } | 731 | } |
732 | 732 | ||
733 | extern void uv_bau_message_intr1(void); | 733 | extern void uv_bau_message_intr1(void); |
734 | #ifdef CONFIG_TRACING | ||
735 | #define trace_uv_bau_message_intr1 uv_bau_message_intr1 | ||
736 | #endif | ||
734 | extern void uv_bau_timeout_intr1(void); | 737 | extern void uv_bau_timeout_intr1(void); |
735 | 738 | ||
736 | struct atomic_short { | 739 | struct atomic_short { |
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 7bd3bd310106..74b3891afb9b 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile | |||
@@ -102,6 +102,7 @@ obj-$(CONFIG_OF) += devicetree.o | |||
102 | obj-$(CONFIG_UPROBES) += uprobes.o | 102 | obj-$(CONFIG_UPROBES) += uprobes.o |
103 | 103 | ||
104 | obj-$(CONFIG_PERF_EVENTS) += perf_regs.o | 104 | obj-$(CONFIG_PERF_EVENTS) += perf_regs.o |
105 | obj-$(CONFIG_TRACING) += tracepoint.o | ||
105 | 106 | ||
106 | ### | 107 | ### |
107 | # 64 bit specific files | 108 | # 64 bit specific files |
diff --git a/arch/x86/kernel/apic/Makefile b/arch/x86/kernel/apic/Makefile index 0ae0323b1f9c..5274c3a07e75 100644 --- a/arch/x86/kernel/apic/Makefile +++ b/arch/x86/kernel/apic/Makefile | |||
@@ -2,6 +2,7 @@ | |||
2 | # Makefile for local APIC drivers and for the IO-APIC code | 2 | # Makefile for local APIC drivers and for the IO-APIC code |
3 | # | 3 | # |
4 | 4 | ||
5 | CFLAGS_apic.o := -I$(src)/../../include/asm/trace | ||
5 | obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_noop.o ipi.o | 6 | obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_noop.o ipi.o |
6 | obj-y += hw_nmi.o | 7 | obj-y += hw_nmi.o |
7 | 8 | ||
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 59ee76fe1c53..61ced40e9c2c 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c | |||
@@ -55,6 +55,9 @@ | |||
55 | #include <asm/tsc.h> | 55 | #include <asm/tsc.h> |
56 | #include <asm/hypervisor.h> | 56 | #include <asm/hypervisor.h> |
57 | 57 | ||
58 | #define CREATE_TRACE_POINTS | ||
59 | #include <asm/trace/irq_vectors.h> | ||
60 | |||
58 | unsigned int num_processors; | 61 | unsigned int num_processors; |
59 | 62 | ||
60 | unsigned disabled_cpus __cpuinitdata; | 63 | unsigned disabled_cpus __cpuinitdata; |
@@ -931,6 +934,27 @@ void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs) | |||
931 | set_irq_regs(old_regs); | 934 | set_irq_regs(old_regs); |
932 | } | 935 | } |
933 | 936 | ||
937 | void __irq_entry smp_trace_apic_timer_interrupt(struct pt_regs *regs) | ||
938 | { | ||
939 | struct pt_regs *old_regs = set_irq_regs(regs); | ||
940 | |||
941 | /* | ||
942 | * NOTE! We'd better ACK the irq immediately, | ||
943 | * because timer handling can be slow. | ||
944 | * | ||
945 | * update_process_times() expects us to have done irq_enter(). | ||
946 | * Besides, if we don't timer interrupts ignore the global | ||
947 | * interrupt lock, which is the WrongThing (tm) to do. | ||
948 | */ | ||
949 | entering_ack_irq(); | ||
950 | trace_local_timer_entry(LOCAL_TIMER_VECTOR); | ||
951 | local_apic_timer_interrupt(); | ||
952 | trace_local_timer_exit(LOCAL_TIMER_VECTOR); | ||
953 | exiting_irq(); | ||
954 | |||
955 | set_irq_regs(old_regs); | ||
956 | } | ||
957 | |||
934 | int setup_profiling_timer(unsigned int multiplier) | 958 | int setup_profiling_timer(unsigned int multiplier) |
935 | { | 959 | { |
936 | return -EINVAL; | 960 | return -EINVAL; |
@@ -1931,6 +1955,15 @@ void smp_spurious_interrupt(struct pt_regs *regs) | |||
1931 | exiting_irq(); | 1955 | exiting_irq(); |
1932 | } | 1956 | } |
1933 | 1957 | ||
1958 | void smp_trace_spurious_interrupt(struct pt_regs *regs) | ||
1959 | { | ||
1960 | entering_irq(); | ||
1961 | trace_spurious_apic_entry(SPURIOUS_APIC_VECTOR); | ||
1962 | __smp_spurious_interrupt(); | ||
1963 | trace_spurious_apic_exit(SPURIOUS_APIC_VECTOR); | ||
1964 | exiting_irq(); | ||
1965 | } | ||
1966 | |||
1934 | /* | 1967 | /* |
1935 | * This interrupt should never happen with our APIC/SMP architecture | 1968 | * This interrupt should never happen with our APIC/SMP architecture |
1936 | */ | 1969 | */ |
@@ -1978,6 +2011,15 @@ void smp_error_interrupt(struct pt_regs *regs) | |||
1978 | exiting_irq(); | 2011 | exiting_irq(); |
1979 | } | 2012 | } |
1980 | 2013 | ||
2014 | void smp_trace_error_interrupt(struct pt_regs *regs) | ||
2015 | { | ||
2016 | entering_irq(); | ||
2017 | trace_error_apic_entry(ERROR_APIC_VECTOR); | ||
2018 | __smp_error_interrupt(regs); | ||
2019 | trace_error_apic_exit(ERROR_APIC_VECTOR); | ||
2020 | exiting_irq(); | ||
2021 | } | ||
2022 | |||
1981 | /** | 2023 | /** |
1982 | * connect_bsp_APIC - attach the APIC to the interrupt system | 2024 | * connect_bsp_APIC - attach the APIC to the interrupt system |
1983 | */ | 2025 | */ |
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 8f6a0f909d6f..d9c8c0947ec2 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c | |||
@@ -1257,7 +1257,7 @@ void __cpuinit cpu_init(void) | |||
1257 | switch_to_new_gdt(cpu); | 1257 | switch_to_new_gdt(cpu); |
1258 | loadsegment(fs, 0); | 1258 | loadsegment(fs, 0); |
1259 | 1259 | ||
1260 | load_idt((const struct desc_ptr *)&idt_descr); | 1260 | load_current_idt(); |
1261 | 1261 | ||
1262 | memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8); | 1262 | memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8); |
1263 | syscall_init(); | 1263 | syscall_init(); |
@@ -1334,7 +1334,7 @@ void __cpuinit cpu_init(void) | |||
1334 | if (cpu_has_vme || cpu_has_tsc || cpu_has_de) | 1334 | if (cpu_has_vme || cpu_has_tsc || cpu_has_de) |
1335 | clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE); | 1335 | clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE); |
1336 | 1336 | ||
1337 | load_idt(&idt_descr); | 1337 | load_current_idt(); |
1338 | switch_to_new_gdt(cpu); | 1338 | switch_to_new_gdt(cpu); |
1339 | 1339 | ||
1340 | /* | 1340 | /* |
diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c index f6b35f2a6a37..2f3a7995e56a 100644 --- a/arch/x86/kernel/cpu/mcheck/therm_throt.c +++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c | |||
@@ -29,6 +29,7 @@ | |||
29 | #include <asm/idle.h> | 29 | #include <asm/idle.h> |
30 | #include <asm/mce.h> | 30 | #include <asm/mce.h> |
31 | #include <asm/msr.h> | 31 | #include <asm/msr.h> |
32 | #include <asm/trace/irq_vectors.h> | ||
32 | 33 | ||
33 | /* How long to wait between reporting thermal events */ | 34 | /* How long to wait between reporting thermal events */ |
34 | #define CHECK_INTERVAL (300 * HZ) | 35 | #define CHECK_INTERVAL (300 * HZ) |
@@ -391,6 +392,15 @@ asmlinkage void smp_thermal_interrupt(struct pt_regs *regs) | |||
391 | exiting_ack_irq(); | 392 | exiting_ack_irq(); |
392 | } | 393 | } |
393 | 394 | ||
395 | asmlinkage void smp_trace_thermal_interrupt(struct pt_regs *regs) | ||
396 | { | ||
397 | entering_irq(); | ||
398 | trace_thermal_apic_entry(THERMAL_APIC_VECTOR); | ||
399 | __smp_thermal_interrupt(); | ||
400 | trace_thermal_apic_exit(THERMAL_APIC_VECTOR); | ||
401 | exiting_ack_irq(); | ||
402 | } | ||
403 | |||
394 | /* Thermal monitoring depends on APIC, ACPI and clock modulation */ | 404 | /* Thermal monitoring depends on APIC, ACPI and clock modulation */ |
395 | static int intel_thermal_supported(struct cpuinfo_x86 *c) | 405 | static int intel_thermal_supported(struct cpuinfo_x86 *c) |
396 | { | 406 | { |
diff --git a/arch/x86/kernel/cpu/mcheck/threshold.c b/arch/x86/kernel/cpu/mcheck/threshold.c index 610cd98d6ef9..fe6b1c86645b 100644 --- a/arch/x86/kernel/cpu/mcheck/threshold.c +++ b/arch/x86/kernel/cpu/mcheck/threshold.c | |||
@@ -8,6 +8,7 @@ | |||
8 | #include <asm/apic.h> | 8 | #include <asm/apic.h> |
9 | #include <asm/idle.h> | 9 | #include <asm/idle.h> |
10 | #include <asm/mce.h> | 10 | #include <asm/mce.h> |
11 | #include <asm/trace/irq_vectors.h> | ||
11 | 12 | ||
12 | static void default_threshold_interrupt(void) | 13 | static void default_threshold_interrupt(void) |
13 | { | 14 | { |
@@ -29,3 +30,12 @@ asmlinkage void smp_threshold_interrupt(void) | |||
29 | __smp_threshold_interrupt(); | 30 | __smp_threshold_interrupt(); |
30 | exiting_ack_irq(); | 31 | exiting_ack_irq(); |
31 | } | 32 | } |
33 | |||
34 | asmlinkage void smp_trace_threshold_interrupt(void) | ||
35 | { | ||
36 | entering_irq(); | ||
37 | trace_threshold_apic_entry(THRESHOLD_APIC_VECTOR); | ||
38 | __smp_threshold_interrupt(); | ||
39 | trace_threshold_apic_exit(THRESHOLD_APIC_VECTOR); | ||
40 | exiting_ack_irq(); | ||
41 | } | ||
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S index 8f3e2dec1df3..2cfbc3a3a2dd 100644 --- a/arch/x86/kernel/entry_32.S +++ b/arch/x86/kernel/entry_32.S | |||
@@ -801,7 +801,17 @@ ENTRY(name) \ | |||
801 | CFI_ENDPROC; \ | 801 | CFI_ENDPROC; \ |
802 | ENDPROC(name) | 802 | ENDPROC(name) |
803 | 803 | ||
804 | #define BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(name, nr, smp_##name) | 804 | |
805 | #ifdef CONFIG_TRACING | ||
806 | #define TRACE_BUILD_INTERRUPT(name, nr) \ | ||
807 | BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name) | ||
808 | #else | ||
809 | #define TRACE_BUILD_INTERRUPT(name, nr) | ||
810 | #endif | ||
811 | |||
812 | #define BUILD_INTERRUPT(name, nr) \ | ||
813 | BUILD_INTERRUPT3(name, nr, smp_##name); \ | ||
814 | TRACE_BUILD_INTERRUPT(name, nr) | ||
805 | 815 | ||
806 | /* The include is where all of the SMP etc. interrupts come from */ | 816 | /* The include is where all of the SMP etc. interrupts come from */ |
807 | #include <asm/entry_arch.h> | 817 | #include <asm/entry_arch.h> |
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 727208941030..11eef43f971f 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S | |||
@@ -1138,7 +1138,7 @@ END(common_interrupt) | |||
1138 | /* | 1138 | /* |
1139 | * APIC interrupts. | 1139 | * APIC interrupts. |
1140 | */ | 1140 | */ |
1141 | .macro apicinterrupt num sym do_sym | 1141 | .macro apicinterrupt3 num sym do_sym |
1142 | ENTRY(\sym) | 1142 | ENTRY(\sym) |
1143 | INTR_FRAME | 1143 | INTR_FRAME |
1144 | ASM_CLAC | 1144 | ASM_CLAC |
@@ -1150,15 +1150,32 @@ ENTRY(\sym) | |||
1150 | END(\sym) | 1150 | END(\sym) |
1151 | .endm | 1151 | .endm |
1152 | 1152 | ||
1153 | #ifdef CONFIG_TRACING | ||
1154 | #define trace(sym) trace_##sym | ||
1155 | #define smp_trace(sym) smp_trace_##sym | ||
1156 | |||
1157 | .macro trace_apicinterrupt num sym | ||
1158 | apicinterrupt3 \num trace(\sym) smp_trace(\sym) | ||
1159 | .endm | ||
1160 | #else | ||
1161 | .macro trace_apicinterrupt num sym do_sym | ||
1162 | .endm | ||
1163 | #endif | ||
1164 | |||
1165 | .macro apicinterrupt num sym do_sym | ||
1166 | apicinterrupt3 \num \sym \do_sym | ||
1167 | trace_apicinterrupt \num \sym | ||
1168 | .endm | ||
1169 | |||
1153 | #ifdef CONFIG_SMP | 1170 | #ifdef CONFIG_SMP |
1154 | apicinterrupt IRQ_MOVE_CLEANUP_VECTOR \ | 1171 | apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR \ |
1155 | irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt | 1172 | irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt |
1156 | apicinterrupt REBOOT_VECTOR \ | 1173 | apicinterrupt3 REBOOT_VECTOR \ |
1157 | reboot_interrupt smp_reboot_interrupt | 1174 | reboot_interrupt smp_reboot_interrupt |
1158 | #endif | 1175 | #endif |
1159 | 1176 | ||
1160 | #ifdef CONFIG_X86_UV | 1177 | #ifdef CONFIG_X86_UV |
1161 | apicinterrupt UV_BAU_MESSAGE \ | 1178 | apicinterrupt3 UV_BAU_MESSAGE \ |
1162 | uv_bau_message_intr1 uv_bau_message_interrupt | 1179 | uv_bau_message_intr1 uv_bau_message_interrupt |
1163 | #endif | 1180 | #endif |
1164 | apicinterrupt LOCAL_TIMER_VECTOR \ | 1181 | apicinterrupt LOCAL_TIMER_VECTOR \ |
@@ -1167,7 +1184,7 @@ apicinterrupt X86_PLATFORM_IPI_VECTOR \ | |||
1167 | x86_platform_ipi smp_x86_platform_ipi | 1184 | x86_platform_ipi smp_x86_platform_ipi |
1168 | 1185 | ||
1169 | #ifdef CONFIG_HAVE_KVM | 1186 | #ifdef CONFIG_HAVE_KVM |
1170 | apicinterrupt POSTED_INTR_VECTOR \ | 1187 | apicinterrupt3 POSTED_INTR_VECTOR \ |
1171 | kvm_posted_intr_ipi smp_kvm_posted_intr_ipi | 1188 | kvm_posted_intr_ipi smp_kvm_posted_intr_ipi |
1172 | #endif | 1189 | #endif |
1173 | 1190 | ||
@@ -1451,13 +1468,13 @@ ENTRY(xen_failsafe_callback) | |||
1451 | CFI_ENDPROC | 1468 | CFI_ENDPROC |
1452 | END(xen_failsafe_callback) | 1469 | END(xen_failsafe_callback) |
1453 | 1470 | ||
1454 | apicinterrupt HYPERVISOR_CALLBACK_VECTOR \ | 1471 | apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ |
1455 | xen_hvm_callback_vector xen_evtchn_do_upcall | 1472 | xen_hvm_callback_vector xen_evtchn_do_upcall |
1456 | 1473 | ||
1457 | #endif /* CONFIG_XEN */ | 1474 | #endif /* CONFIG_XEN */ |
1458 | 1475 | ||
1459 | #if IS_ENABLED(CONFIG_HYPERV) | 1476 | #if IS_ENABLED(CONFIG_HYPERV) |
1460 | apicinterrupt HYPERVISOR_CALLBACK_VECTOR \ | 1477 | apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ |
1461 | hyperv_callback_vector hyperv_vector_handler | 1478 | hyperv_callback_vector hyperv_vector_handler |
1462 | #endif /* CONFIG_HYPERV */ | 1479 | #endif /* CONFIG_HYPERV */ |
1463 | 1480 | ||
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 84fb779d5b74..5e4d8a8a5c40 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S | |||
@@ -521,6 +521,12 @@ ENTRY(idt_table) | |||
521 | ENTRY(debug_idt_table) | 521 | ENTRY(debug_idt_table) |
522 | .skip IDT_ENTRIES * 16 | 522 | .skip IDT_ENTRIES * 16 |
523 | 523 | ||
524 | #ifdef CONFIG_TRACING | ||
525 | .align L1_CACHE_BYTES | ||
526 | ENTRY(trace_idt_table) | ||
527 | .skip IDT_ENTRIES * 16 | ||
528 | #endif | ||
529 | |||
524 | __PAGE_ALIGNED_BSS | 530 | __PAGE_ALIGNED_BSS |
525 | NEXT_PAGE(empty_zero_page) | 531 | NEXT_PAGE(empty_zero_page) |
526 | .skip PAGE_SIZE | 532 | .skip PAGE_SIZE |
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index e3b8df1754cc..06af119743a6 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c | |||
@@ -17,6 +17,7 @@ | |||
17 | #include <asm/idle.h> | 17 | #include <asm/idle.h> |
18 | #include <asm/mce.h> | 18 | #include <asm/mce.h> |
19 | #include <asm/hw_irq.h> | 19 | #include <asm/hw_irq.h> |
20 | #include <asm/trace/irq_vectors.h> | ||
20 | 21 | ||
21 | atomic_t irq_err_count; | 22 | atomic_t irq_err_count; |
22 | 23 | ||
@@ -244,6 +245,18 @@ void smp_kvm_posted_intr_ipi(struct pt_regs *regs) | |||
244 | } | 245 | } |
245 | #endif | 246 | #endif |
246 | 247 | ||
248 | void smp_trace_x86_platform_ipi(struct pt_regs *regs) | ||
249 | { | ||
250 | struct pt_regs *old_regs = set_irq_regs(regs); | ||
251 | |||
252 | entering_ack_irq(); | ||
253 | trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR); | ||
254 | __smp_x86_platform_ipi(); | ||
255 | trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR); | ||
256 | exiting_irq(); | ||
257 | set_irq_regs(old_regs); | ||
258 | } | ||
259 | |||
247 | EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq); | 260 | EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq); |
248 | 261 | ||
249 | #ifdef CONFIG_HOTPLUG_CPU | 262 | #ifdef CONFIG_HOTPLUG_CPU |
diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c index 074d46fdbd1f..636a55e4a13c 100644 --- a/arch/x86/kernel/irq_work.c +++ b/arch/x86/kernel/irq_work.c | |||
@@ -8,6 +8,7 @@ | |||
8 | #include <linux/irq_work.h> | 8 | #include <linux/irq_work.h> |
9 | #include <linux/hardirq.h> | 9 | #include <linux/hardirq.h> |
10 | #include <asm/apic.h> | 10 | #include <asm/apic.h> |
11 | #include <asm/trace/irq_vectors.h> | ||
11 | 12 | ||
12 | static inline void irq_work_entering_irq(void) | 13 | static inline void irq_work_entering_irq(void) |
13 | { | 14 | { |
@@ -28,6 +29,15 @@ void smp_irq_work_interrupt(struct pt_regs *regs) | |||
28 | exiting_irq(); | 29 | exiting_irq(); |
29 | } | 30 | } |
30 | 31 | ||
32 | void smp_trace_irq_work_interrupt(struct pt_regs *regs) | ||
33 | { | ||
34 | irq_work_entering_irq(); | ||
35 | trace_irq_work_entry(IRQ_WORK_VECTOR); | ||
36 | __smp_irq_work_interrupt(); | ||
37 | trace_irq_work_exit(IRQ_WORK_VECTOR); | ||
38 | exiting_irq(); | ||
39 | } | ||
40 | |||
31 | void arch_irq_work_raise(void) | 41 | void arch_irq_work_raise(void) |
32 | { | 42 | { |
33 | #ifdef CONFIG_X86_LOCAL_APIC | 43 | #ifdef CONFIG_X86_LOCAL_APIC |
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index d85837574a79..f4fe0b8879e0 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c | |||
@@ -30,6 +30,7 @@ | |||
30 | #include <asm/proto.h> | 30 | #include <asm/proto.h> |
31 | #include <asm/apic.h> | 31 | #include <asm/apic.h> |
32 | #include <asm/nmi.h> | 32 | #include <asm/nmi.h> |
33 | #include <asm/trace/irq_vectors.h> | ||
33 | /* | 34 | /* |
34 | * Some notes on x86 processor bugs affecting SMP operation: | 35 | * Some notes on x86 processor bugs affecting SMP operation: |
35 | * | 36 | * |
@@ -264,6 +265,17 @@ void smp_reschedule_interrupt(struct pt_regs *regs) | |||
264 | */ | 265 | */ |
265 | } | 266 | } |
266 | 267 | ||
268 | void smp_trace_reschedule_interrupt(struct pt_regs *regs) | ||
269 | { | ||
270 | ack_APIC_irq(); | ||
271 | trace_reschedule_entry(RESCHEDULE_VECTOR); | ||
272 | __smp_reschedule_interrupt(); | ||
273 | trace_reschedule_exit(RESCHEDULE_VECTOR); | ||
274 | /* | ||
275 | * KVM uses this interrupt to force a cpu out of guest mode | ||
276 | */ | ||
277 | } | ||
278 | |||
267 | static inline void call_function_entering_irq(void) | 279 | static inline void call_function_entering_irq(void) |
268 | { | 280 | { |
269 | ack_APIC_irq(); | 281 | ack_APIC_irq(); |
@@ -283,6 +295,15 @@ void smp_call_function_interrupt(struct pt_regs *regs) | |||
283 | exiting_irq(); | 295 | exiting_irq(); |
284 | } | 296 | } |
285 | 297 | ||
298 | void smp_trace_call_function_interrupt(struct pt_regs *regs) | ||
299 | { | ||
300 | call_function_entering_irq(); | ||
301 | trace_call_function_entry(CALL_FUNCTION_VECTOR); | ||
302 | __smp_call_function_interrupt(); | ||
303 | trace_call_function_exit(CALL_FUNCTION_VECTOR); | ||
304 | exiting_irq(); | ||
305 | } | ||
306 | |||
286 | static inline void __smp_call_function_single_interrupt(void) | 307 | static inline void __smp_call_function_single_interrupt(void) |
287 | { | 308 | { |
288 | generic_smp_call_function_single_interrupt(); | 309 | generic_smp_call_function_single_interrupt(); |
@@ -296,6 +317,15 @@ void smp_call_function_single_interrupt(struct pt_regs *regs) | |||
296 | exiting_irq(); | 317 | exiting_irq(); |
297 | } | 318 | } |
298 | 319 | ||
320 | void smp_trace_call_function_single_interrupt(struct pt_regs *regs) | ||
321 | { | ||
322 | call_function_entering_irq(); | ||
323 | trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR); | ||
324 | __smp_call_function_single_interrupt(); | ||
325 | trace_call_function_single_exit(CALL_FUNCTION_SINGLE_VECTOR); | ||
326 | exiting_irq(); | ||
327 | } | ||
328 | |||
299 | static int __init nonmi_ipi_setup(char *str) | 329 | static int __init nonmi_ipi_setup(char *str) |
300 | { | 330 | { |
301 | smp_no_nmi_ipi = true; | 331 | smp_no_nmi_ipi = true; |
diff --git a/arch/x86/kernel/tracepoint.c b/arch/x86/kernel/tracepoint.c new file mode 100644 index 000000000000..1423efe98fbc --- /dev/null +++ b/arch/x86/kernel/tracepoint.c | |||
@@ -0,0 +1,57 @@ | |||
1 | /* | ||
2 | * Code for supporting irq vector tracepoints. | ||
3 | * | ||
4 | * Copyright (C) 2013 Seiji Aguchi <seiji.aguchi@hds.com> | ||
5 | * | ||
6 | */ | ||
7 | #include <asm/hw_irq.h> | ||
8 | #include <asm/desc.h> | ||
9 | #include <linux/atomic.h> | ||
10 | |||
11 | atomic_t trace_idt_ctr = ATOMIC_INIT(0); | ||
12 | struct desc_ptr trace_idt_descr = { NR_VECTORS * 16 - 1, | ||
13 | (unsigned long) trace_idt_table }; | ||
14 | |||
15 | #ifndef CONFIG_X86_64 | ||
16 | gate_desc trace_idt_table[NR_VECTORS] __page_aligned_data | ||
17 | = { { { { 0, 0 } } }, }; | ||
18 | #endif | ||
19 | |||
20 | static int trace_irq_vector_refcount; | ||
21 | static DEFINE_MUTEX(irq_vector_mutex); | ||
22 | |||
23 | static void set_trace_idt_ctr(int val) | ||
24 | { | ||
25 | atomic_set(&trace_idt_ctr, val); | ||
26 | /* Ensure the trace_idt_ctr is set before sending IPI */ | ||
27 | wmb(); | ||
28 | } | ||
29 | |||
30 | static void switch_idt(void *arg) | ||
31 | { | ||
32 | load_current_idt(); | ||
33 | } | ||
34 | |||
35 | void trace_irq_vector_regfunc(void) | ||
36 | { | ||
37 | mutex_lock(&irq_vector_mutex); | ||
38 | if (!trace_irq_vector_refcount) { | ||
39 | set_trace_idt_ctr(1); | ||
40 | smp_call_function(switch_idt, NULL, 0); | ||
41 | switch_idt(NULL); | ||
42 | } | ||
43 | trace_irq_vector_refcount++; | ||
44 | mutex_unlock(&irq_vector_mutex); | ||
45 | } | ||
46 | |||
47 | void trace_irq_vector_unregfunc(void) | ||
48 | { | ||
49 | mutex_lock(&irq_vector_mutex); | ||
50 | trace_irq_vector_refcount--; | ||
51 | if (!trace_irq_vector_refcount) { | ||
52 | set_trace_idt_ctr(0); | ||
53 | smp_call_function(switch_idt, NULL, 0); | ||
54 | switch_idt(NULL); | ||
55 | } | ||
56 | mutex_unlock(&irq_vector_mutex); | ||
57 | } | ||
diff --git a/include/xen/events.h b/include/xen/events.h index b2b27c6a0f7b..c9ea10ee2273 100644 --- a/include/xen/events.h +++ b/include/xen/events.h | |||
@@ -76,6 +76,9 @@ unsigned irq_from_evtchn(unsigned int evtchn); | |||
76 | 76 | ||
77 | /* Xen HVM evtchn vector callback */ | 77 | /* Xen HVM evtchn vector callback */ |
78 | void xen_hvm_callback_vector(void); | 78 | void xen_hvm_callback_vector(void); |
79 | #ifdef CONFIG_TRACING | ||
80 | #define trace_xen_hvm_callback_vector xen_hvm_callback_vector | ||
81 | #endif | ||
79 | extern int xen_have_vector_callback; | 82 | extern int xen_have_vector_callback; |
80 | int xen_set_callback_via(uint64_t via); | 83 | int xen_set_callback_via(uint64_t via); |
81 | void xen_evtchn_do_upcall(struct pt_regs *regs); | 84 | void xen_evtchn_do_upcall(struct pt_regs *regs); |