diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2011-05-19 20:36:08 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2011-05-19 20:36:08 -0400 |
commit | df48d8716eab9608fe93924e4ae06ff110e8674f (patch) | |
tree | 0fe10733a414b3651e1dae29518b7960a4da0aa4 | |
parent | acd30250d7d0f495685d1c7c6184636a22fcdf7f (diff) | |
parent | 29510ec3b626c86de9707bb8904ff940d430289b (diff) |
Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (107 commits)
perf stat: Add more cache-miss percentage printouts
perf stat: Add -d -d and -d -d -d options to show more CPU events
ftrace/kbuild: Add recordmcount files to force full build
ftrace: Add self-tests for multiple function trace users
ftrace: Modify ftrace_set_filter/notrace to take ops
ftrace: Allow dynamically allocated function tracers
ftrace: Implement separate user function filtering
ftrace: Free hash with call_rcu_sched()
ftrace: Have global_ops store the functions that are to be traced
ftrace: Add ops parameter to ftrace_startup/shutdown functions
ftrace: Add enabled_functions file
ftrace: Use counters to enable functions to trace
ftrace: Separate hash allocation and assignment
ftrace: Create a global_ops to hold the filter and notrace hashes
ftrace: Use hash instead for FTRACE_FL_FILTER
ftrace: Replace FTRACE_FL_NOTRACE flag with a hash of ignored functions
perf bench, x86: Add alternatives-asm.h wrapper
x86, 64-bit: Fix copy_[to/from]_user() checks for the userspace address limit
x86, mem: memset_64.S: Optimize memset by enhanced REP MOVSB/STOSB
x86, mem: memmove_64.S: Optimize memmove by enhanced REP MOVSB/STOSB
...
78 files changed, 4013 insertions, 1700 deletions
@@ -1268,6 +1268,7 @@ help: | |||
1268 | @echo ' make C=1 [targets] Check all c source with $$CHECK (sparse by default)' | 1268 | @echo ' make C=1 [targets] Check all c source with $$CHECK (sparse by default)' |
1269 | @echo ' make C=2 [targets] Force check of all c source with $$CHECK' | 1269 | @echo ' make C=2 [targets] Force check of all c source with $$CHECK' |
1270 | @echo ' make W=1 [targets] Enable extra gcc checks' | 1270 | @echo ' make W=1 [targets] Enable extra gcc checks' |
1271 | @echo ' make RECORDMCOUNT_WARN=1 [targets] Warn about ignored mcount sections' | ||
1271 | @echo '' | 1272 | @echo '' |
1272 | @echo 'Execute "make" or "make all" to build all targets marked with [*] ' | 1273 | @echo 'Execute "make" or "make all" to build all targets marked with [*] ' |
1273 | @echo 'For further info see the ./README file' | 1274 | @echo 'For further info see the ./README file' |
diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h index 7622ccf75076..1881b316ca45 100644 --- a/arch/mips/include/asm/jump_label.h +++ b/arch/mips/include/asm/jump_label.h | |||
@@ -20,16 +20,18 @@ | |||
20 | #define WORD_INSN ".word" | 20 | #define WORD_INSN ".word" |
21 | #endif | 21 | #endif |
22 | 22 | ||
23 | #define JUMP_LABEL(key, label) \ | 23 | static __always_inline bool arch_static_branch(struct jump_label_key *key) |
24 | do { \ | 24 | { |
25 | asm goto("1:\tnop\n\t" \ | 25 | asm goto("1:\tnop\n\t" |
26 | "nop\n\t" \ | 26 | "nop\n\t" |
27 | ".pushsection __jump_table, \"a\"\n\t" \ | 27 | ".pushsection __jump_table, \"aw\"\n\t" |
28 | WORD_INSN " 1b, %l[" #label "], %0\n\t" \ | 28 | WORD_INSN " 1b, %l[l_yes], %0\n\t" |
29 | ".popsection\n\t" \ | 29 | ".popsection\n\t" |
30 | : : "i" (key) : : label); \ | 30 | : : "i" (key) : : l_yes); |
31 | } while (0) | 31 | return false; |
32 | 32 | l_yes: | |
33 | return true; | ||
34 | } | ||
33 | 35 | ||
34 | #endif /* __KERNEL__ */ | 36 | #endif /* __KERNEL__ */ |
35 | 37 | ||
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 2508a6f31588..4a7f14079e03 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig | |||
@@ -88,6 +88,7 @@ config S390 | |||
88 | select HAVE_KERNEL_XZ | 88 | select HAVE_KERNEL_XZ |
89 | select HAVE_GET_USER_PAGES_FAST | 89 | select HAVE_GET_USER_PAGES_FAST |
90 | select HAVE_ARCH_MUTEX_CPU_RELAX | 90 | select HAVE_ARCH_MUTEX_CPU_RELAX |
91 | select HAVE_ARCH_JUMP_LABEL if !MARCH_G5 | ||
91 | select ARCH_INLINE_SPIN_TRYLOCK | 92 | select ARCH_INLINE_SPIN_TRYLOCK |
92 | select ARCH_INLINE_SPIN_TRYLOCK_BH | 93 | select ARCH_INLINE_SPIN_TRYLOCK_BH |
93 | select ARCH_INLINE_SPIN_LOCK | 94 | select ARCH_INLINE_SPIN_LOCK |
diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index 3c29be4836ed..b7931faaef6d 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h | |||
@@ -11,15 +11,13 @@ struct dyn_arch_ftrace { }; | |||
11 | 11 | ||
12 | #ifdef CONFIG_64BIT | 12 | #ifdef CONFIG_64BIT |
13 | #define MCOUNT_INSN_SIZE 12 | 13 | #define MCOUNT_INSN_SIZE 12 |
14 | #define MCOUNT_OFFSET 8 | ||
15 | #else | 14 | #else |
16 | #define MCOUNT_INSN_SIZE 20 | 15 | #define MCOUNT_INSN_SIZE 20 |
17 | #define MCOUNT_OFFSET 4 | ||
18 | #endif | 16 | #endif |
19 | 17 | ||
20 | static inline unsigned long ftrace_call_adjust(unsigned long addr) | 18 | static inline unsigned long ftrace_call_adjust(unsigned long addr) |
21 | { | 19 | { |
22 | return addr - MCOUNT_OFFSET; | 20 | return addr; |
23 | } | 21 | } |
24 | 22 | ||
25 | #endif /* __ASSEMBLY__ */ | 23 | #endif /* __ASSEMBLY__ */ |
diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h new file mode 100644 index 000000000000..95a6cf2b5b67 --- /dev/null +++ b/arch/s390/include/asm/jump_label.h | |||
@@ -0,0 +1,37 @@ | |||
1 | #ifndef _ASM_S390_JUMP_LABEL_H | ||
2 | #define _ASM_S390_JUMP_LABEL_H | ||
3 | |||
4 | #include <linux/types.h> | ||
5 | |||
6 | #define JUMP_LABEL_NOP_SIZE 6 | ||
7 | |||
8 | #ifdef CONFIG_64BIT | ||
9 | #define ASM_PTR ".quad" | ||
10 | #define ASM_ALIGN ".balign 8" | ||
11 | #else | ||
12 | #define ASM_PTR ".long" | ||
13 | #define ASM_ALIGN ".balign 4" | ||
14 | #endif | ||
15 | |||
16 | static __always_inline bool arch_static_branch(struct jump_label_key *key) | ||
17 | { | ||
18 | asm goto("0: brcl 0,0\n" | ||
19 | ".pushsection __jump_table, \"aw\"\n" | ||
20 | ASM_ALIGN "\n" | ||
21 | ASM_PTR " 0b, %l[label], %0\n" | ||
22 | ".popsection\n" | ||
23 | : : "X" (key) : : label); | ||
24 | return false; | ||
25 | label: | ||
26 | return true; | ||
27 | } | ||
28 | |||
29 | typedef unsigned long jump_label_t; | ||
30 | |||
31 | struct jump_entry { | ||
32 | jump_label_t code; | ||
33 | jump_label_t target; | ||
34 | jump_label_t key; | ||
35 | }; | ||
36 | |||
37 | #endif | ||
diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 64230bc392fa..5ff15dacb571 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile | |||
@@ -23,7 +23,7 @@ CFLAGS_sysinfo.o += -Iinclude/math-emu -Iarch/s390/math-emu -w | |||
23 | obj-y := bitmap.o traps.o time.o process.o base.o early.o setup.o \ | 23 | obj-y := bitmap.o traps.o time.o process.o base.o early.o setup.o \ |
24 | processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o \ | 24 | processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o \ |
25 | s390_ext.o debug.o irq.o ipl.o dis.o diag.o mem_detect.o \ | 25 | s390_ext.o debug.o irq.o ipl.o dis.o diag.o mem_detect.o \ |
26 | vdso.o vtime.o sysinfo.o nmi.o sclp.o | 26 | vdso.o vtime.o sysinfo.o nmi.o sclp.o jump_label.o |
27 | 27 | ||
28 | obj-y += $(if $(CONFIG_64BIT),entry64.o,entry.o) | 28 | obj-y += $(if $(CONFIG_64BIT),entry64.o,entry.o) |
29 | obj-y += $(if $(CONFIG_64BIT),reipl64.o,reipl.o) | 29 | obj-y += $(if $(CONFIG_64BIT),reipl64.o,reipl.o) |
diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c new file mode 100644 index 000000000000..44cc06bedf77 --- /dev/null +++ b/arch/s390/kernel/jump_label.c | |||
@@ -0,0 +1,59 @@ | |||
1 | /* | ||
2 | * Jump label s390 support | ||
3 | * | ||
4 | * Copyright IBM Corp. 2011 | ||
5 | * Author(s): Jan Glauber <jang@linux.vnet.ibm.com> | ||
6 | */ | ||
7 | #include <linux/module.h> | ||
8 | #include <linux/uaccess.h> | ||
9 | #include <linux/stop_machine.h> | ||
10 | #include <linux/jump_label.h> | ||
11 | #include <asm/ipl.h> | ||
12 | |||
13 | #ifdef HAVE_JUMP_LABEL | ||
14 | |||
15 | struct insn { | ||
16 | u16 opcode; | ||
17 | s32 offset; | ||
18 | } __packed; | ||
19 | |||
20 | struct insn_args { | ||
21 | unsigned long *target; | ||
22 | struct insn *insn; | ||
23 | ssize_t size; | ||
24 | }; | ||
25 | |||
26 | static int __arch_jump_label_transform(void *data) | ||
27 | { | ||
28 | struct insn_args *args = data; | ||
29 | int rc; | ||
30 | |||
31 | rc = probe_kernel_write(args->target, args->insn, args->size); | ||
32 | WARN_ON_ONCE(rc < 0); | ||
33 | return 0; | ||
34 | } | ||
35 | |||
36 | void arch_jump_label_transform(struct jump_entry *entry, | ||
37 | enum jump_label_type type) | ||
38 | { | ||
39 | struct insn_args args; | ||
40 | struct insn insn; | ||
41 | |||
42 | if (type == JUMP_LABEL_ENABLE) { | ||
43 | /* brcl 15,offset */ | ||
44 | insn.opcode = 0xc0f4; | ||
45 | insn.offset = (entry->target - entry->code) >> 1; | ||
46 | } else { | ||
47 | /* brcl 0,0 */ | ||
48 | insn.opcode = 0xc004; | ||
49 | insn.offset = 0; | ||
50 | } | ||
51 | |||
52 | args.target = (void *) entry->code; | ||
53 | args.insn = &insn; | ||
54 | args.size = JUMP_LABEL_NOP_SIZE; | ||
55 | |||
56 | stop_machine(__arch_jump_label_transform, &args, NULL); | ||
57 | } | ||
58 | |||
59 | #endif | ||
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h index 427d4684e0d2..fc73a82366f8 100644 --- a/arch/sparc/include/asm/jump_label.h +++ b/arch/sparc/include/asm/jump_label.h | |||
@@ -7,17 +7,20 @@ | |||
7 | 7 | ||
8 | #define JUMP_LABEL_NOP_SIZE 4 | 8 | #define JUMP_LABEL_NOP_SIZE 4 |
9 | 9 | ||
10 | #define JUMP_LABEL(key, label) \ | 10 | static __always_inline bool arch_static_branch(struct jump_label_key *key) |
11 | do { \ | 11 | { |
12 | asm goto("1:\n\t" \ | 12 | asm goto("1:\n\t" |
13 | "nop\n\t" \ | 13 | "nop\n\t" |
14 | "nop\n\t" \ | 14 | "nop\n\t" |
15 | ".pushsection __jump_table, \"a\"\n\t"\ | 15 | ".pushsection __jump_table, \"aw\"\n\t" |
16 | ".align 4\n\t" \ | 16 | ".align 4\n\t" |
17 | ".word 1b, %l[" #label "], %c0\n\t" \ | 17 | ".word 1b, %l[l_yes], %c0\n\t" |
18 | ".popsection \n\t" \ | 18 | ".popsection \n\t" |
19 | : : "i" (key) : : label);\ | 19 | : : "i" (key) : : l_yes); |
20 | } while (0) | 20 | return false; |
21 | l_yes: | ||
22 | return true; | ||
23 | } | ||
21 | 24 | ||
22 | #endif /* __KERNEL__ */ | 25 | #endif /* __KERNEL__ */ |
23 | 26 | ||
diff --git a/arch/x86/include/asm/alternative-asm.h b/arch/x86/include/asm/alternative-asm.h index a63a68be1cce..94d420b360d1 100644 --- a/arch/x86/include/asm/alternative-asm.h +++ b/arch/x86/include/asm/alternative-asm.h | |||
@@ -15,4 +15,13 @@ | |||
15 | .endm | 15 | .endm |
16 | #endif | 16 | #endif |
17 | 17 | ||
18 | .macro altinstruction_entry orig alt feature orig_len alt_len | ||
19 | .align 8 | ||
20 | .quad \orig | ||
21 | .quad \alt | ||
22 | .word \feature | ||
23 | .byte \orig_len | ||
24 | .byte \alt_len | ||
25 | .endm | ||
26 | |||
18 | #endif /* __ASSEMBLY__ */ | 27 | #endif /* __ASSEMBLY__ */ |
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index 13009d1af99a..8cdd1e247975 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h | |||
@@ -4,7 +4,6 @@ | |||
4 | #include <linux/types.h> | 4 | #include <linux/types.h> |
5 | #include <linux/stddef.h> | 5 | #include <linux/stddef.h> |
6 | #include <linux/stringify.h> | 6 | #include <linux/stringify.h> |
7 | #include <linux/jump_label.h> | ||
8 | #include <asm/asm.h> | 7 | #include <asm/asm.h> |
9 | 8 | ||
10 | /* | 9 | /* |
@@ -191,7 +190,7 @@ extern void *text_poke(void *addr, const void *opcode, size_t len); | |||
191 | extern void *text_poke_smp(void *addr, const void *opcode, size_t len); | 190 | extern void *text_poke_smp(void *addr, const void *opcode, size_t len); |
192 | extern void text_poke_smp_batch(struct text_poke_param *params, int n); | 191 | extern void text_poke_smp_batch(struct text_poke_param *params, int n); |
193 | 192 | ||
194 | #if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) | 193 | #if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL) |
195 | #define IDEAL_NOP_SIZE_5 5 | 194 | #define IDEAL_NOP_SIZE_5 5 |
196 | extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5]; | 195 | extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5]; |
197 | extern void arch_init_ideal_nop5(void); | 196 | extern void arch_init_ideal_nop5(void); |
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 91f3e087cf21..7f2f7b123293 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h | |||
@@ -195,6 +195,7 @@ | |||
195 | 195 | ||
196 | /* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */ | 196 | /* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */ |
197 | #define X86_FEATURE_FSGSBASE (9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/ | 197 | #define X86_FEATURE_FSGSBASE (9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/ |
198 | #define X86_FEATURE_ERMS (9*32+ 9) /* Enhanced REP MOVSB/STOSB */ | ||
198 | 199 | ||
199 | #if defined(__KERNEL__) && !defined(__ASSEMBLY__) | 200 | #if defined(__KERNEL__) && !defined(__ASSEMBLY__) |
200 | 201 | ||
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index db24c2278be0..268c783ab1c0 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h | |||
@@ -38,11 +38,10 @@ extern void mcount(void); | |||
38 | static inline unsigned long ftrace_call_adjust(unsigned long addr) | 38 | static inline unsigned long ftrace_call_adjust(unsigned long addr) |
39 | { | 39 | { |
40 | /* | 40 | /* |
41 | * call mcount is "e8 <4 byte offset>" | 41 | * addr is the address of the mcount call instruction. |
42 | * The addr points to the 4 byte offset and the caller of this | 42 | * recordmcount does the necessary offset calculation. |
43 | * function wants the pointer to e8. Simply subtract one. | ||
44 | */ | 43 | */ |
45 | return addr - 1; | 44 | return addr; |
46 | } | 45 | } |
47 | 46 | ||
48 | #ifdef CONFIG_DYNAMIC_FTRACE | 47 | #ifdef CONFIG_DYNAMIC_FTRACE |
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index 574dbc22893a..a32b18ce6ead 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h | |||
@@ -5,20 +5,25 @@ | |||
5 | 5 | ||
6 | #include <linux/types.h> | 6 | #include <linux/types.h> |
7 | #include <asm/nops.h> | 7 | #include <asm/nops.h> |
8 | #include <asm/asm.h> | ||
8 | 9 | ||
9 | #define JUMP_LABEL_NOP_SIZE 5 | 10 | #define JUMP_LABEL_NOP_SIZE 5 |
10 | 11 | ||
11 | # define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" | 12 | #define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" |
12 | 13 | ||
13 | # define JUMP_LABEL(key, label) \ | 14 | static __always_inline bool arch_static_branch(struct jump_label_key *key) |
14 | do { \ | 15 | { |
15 | asm goto("1:" \ | 16 | asm goto("1:" |
16 | JUMP_LABEL_INITIAL_NOP \ | 17 | JUMP_LABEL_INITIAL_NOP |
17 | ".pushsection __jump_table, \"aw\" \n\t"\ | 18 | ".pushsection __jump_table, \"aw\" \n\t" |
18 | _ASM_PTR "1b, %l[" #label "], %c0 \n\t" \ | 19 | _ASM_ALIGN "\n\t" |
19 | ".popsection \n\t" \ | 20 | _ASM_PTR "1b, %l[l_yes], %c0 \n\t" |
20 | : : "i" (key) : : label); \ | 21 | ".popsection \n\t" |
21 | } while (0) | 22 | : : "i" (key) : : l_yes); |
23 | return false; | ||
24 | l_yes: | ||
25 | return true; | ||
26 | } | ||
22 | 27 | ||
23 | #endif /* __KERNEL__ */ | 28 | #endif /* __KERNEL__ */ |
24 | 29 | ||
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index db8aa19a08a2..647d8a06ce4f 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h | |||
@@ -88,7 +88,7 @@ void *extend_brk(size_t size, size_t align); | |||
88 | * executable.) | 88 | * executable.) |
89 | */ | 89 | */ |
90 | #define RESERVE_BRK(name,sz) \ | 90 | #define RESERVE_BRK(name,sz) \ |
91 | static void __section(.discard.text) __used \ | 91 | static void __section(.discard.text) __used notrace \ |
92 | __brk_reservation_fn_##name##__(void) { \ | 92 | __brk_reservation_fn_##name##__(void) { \ |
93 | asm volatile ( \ | 93 | asm volatile ( \ |
94 | ".pushsection .brk_reservation,\"aw\",@nobits;" \ | 94 | ".pushsection .brk_reservation,\"aw\",@nobits;" \ |
diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h index d7e89c83645d..70bbe39043a9 100644 --- a/arch/x86/include/asm/stacktrace.h +++ b/arch/x86/include/asm/stacktrace.h | |||
@@ -37,9 +37,6 @@ print_context_stack_bp(struct thread_info *tinfo, | |||
37 | /* Generic stack tracer with callbacks */ | 37 | /* Generic stack tracer with callbacks */ |
38 | 38 | ||
39 | struct stacktrace_ops { | 39 | struct stacktrace_ops { |
40 | void (*warning)(void *data, char *msg); | ||
41 | /* msg must contain %s for the symbol */ | ||
42 | void (*warning_symbol)(void *data, char *msg, unsigned long symbol); | ||
43 | void (*address)(void *data, unsigned long address, int reliable); | 40 | void (*address)(void *data, unsigned long address, int reliable); |
44 | /* On negative return stop dumping */ | 41 | /* On negative return stop dumping */ |
45 | int (*stack)(void *data, char *name); | 42 | int (*stack)(void *data, char *name); |
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index abd3e0ea762a..99f0ad753f32 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h | |||
@@ -42,7 +42,7 @@ | |||
42 | * Returns 0 if the range is valid, nonzero otherwise. | 42 | * Returns 0 if the range is valid, nonzero otherwise. |
43 | * | 43 | * |
44 | * This is equivalent to the following test: | 44 | * This is equivalent to the following test: |
45 | * (u33)addr + (u33)size >= (u33)current->addr_limit.seg (u65 for x86_64) | 45 | * (u33)addr + (u33)size > (u33)current->addr_limit.seg (u65 for x86_64) |
46 | * | 46 | * |
47 | * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry... | 47 | * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry... |
48 | */ | 48 | */ |
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 4a234677e213..1eeeafcb4410 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c | |||
@@ -210,6 +210,15 @@ void __init_or_module apply_alternatives(struct alt_instr *start, | |||
210 | u8 insnbuf[MAX_PATCH_LEN]; | 210 | u8 insnbuf[MAX_PATCH_LEN]; |
211 | 211 | ||
212 | DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); | 212 | DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); |
213 | /* | ||
214 | * The scan order should be from start to end. A later scanned | ||
215 | * alternative code can overwrite a previous scanned alternative code. | ||
216 | * Some kernel functions (e.g. memcpy, memset, etc) use this order to | ||
217 | * patch code. | ||
218 | * | ||
219 | * So be careful if you want to change the scan order to any other | ||
220 | * order. | ||
221 | */ | ||
213 | for (a = start; a < end; a++) { | 222 | for (a = start; a < end; a++) { |
214 | u8 *instr = a->instr; | 223 | u8 *instr = a->instr; |
215 | BUG_ON(a->replacementlen > a->instrlen); | 224 | BUG_ON(a->replacementlen > a->instrlen); |
@@ -679,7 +688,7 @@ void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n) | |||
679 | __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); | 688 | __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); |
680 | } | 689 | } |
681 | 690 | ||
682 | #if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) | 691 | #if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL) |
683 | 692 | ||
684 | #ifdef CONFIG_X86_64 | 693 | #ifdef CONFIG_X86_64 |
685 | unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 }; | 694 | unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 }; |
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index e2ced0074a45..173f3a3fa1a6 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c | |||
@@ -565,8 +565,7 @@ void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c) | |||
565 | 565 | ||
566 | cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx); | 566 | cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx); |
567 | 567 | ||
568 | if (eax > 0) | 568 | c->x86_capability[9] = ebx; |
569 | c->x86_capability[9] = ebx; | ||
570 | } | 569 | } |
571 | 570 | ||
572 | /* AMD-defined flags: level 0x80000001 */ | 571 | /* AMD-defined flags: level 0x80000001 */ |
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index df86bc8c859d..fc73a34ba8c9 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c | |||
@@ -29,10 +29,10 @@ | |||
29 | 29 | ||
30 | static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) | 30 | static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) |
31 | { | 31 | { |
32 | u64 misc_enable; | ||
33 | |||
32 | /* Unmask CPUID levels if masked: */ | 34 | /* Unmask CPUID levels if masked: */ |
33 | if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) { | 35 | if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) { |
34 | u64 misc_enable; | ||
35 | |||
36 | rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable); | 36 | rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable); |
37 | 37 | ||
38 | if (misc_enable & MSR_IA32_MISC_ENABLE_LIMIT_CPUID) { | 38 | if (misc_enable & MSR_IA32_MISC_ENABLE_LIMIT_CPUID) { |
@@ -118,8 +118,6 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) | |||
118 | * (model 2) with the same problem. | 118 | * (model 2) with the same problem. |
119 | */ | 119 | */ |
120 | if (c->x86 == 15) { | 120 | if (c->x86 == 15) { |
121 | u64 misc_enable; | ||
122 | |||
123 | rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable); | 121 | rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable); |
124 | 122 | ||
125 | if (misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING) { | 123 | if (misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING) { |
@@ -130,6 +128,19 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) | |||
130 | } | 128 | } |
131 | } | 129 | } |
132 | #endif | 130 | #endif |
131 | |||
132 | /* | ||
133 | * If fast string is not enabled in IA32_MISC_ENABLE for any reason, | ||
134 | * clear the fast string and enhanced fast string CPU capabilities. | ||
135 | */ | ||
136 | if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) { | ||
137 | rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable); | ||
138 | if (!(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING)) { | ||
139 | printk(KERN_INFO "Disabled fast string operations\n"); | ||
140 | setup_clear_cpu_cap(X86_FEATURE_REP_GOOD); | ||
141 | setup_clear_cpu_cap(X86_FEATURE_ERMS); | ||
142 | } | ||
143 | } | ||
133 | } | 144 | } |
134 | 145 | ||
135 | #ifdef CONFIG_X86_32 | 146 | #ifdef CONFIG_X86_32 |
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index e638689279d3..3a0338b4b179 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c | |||
@@ -31,6 +31,7 @@ | |||
31 | #include <asm/nmi.h> | 31 | #include <asm/nmi.h> |
32 | #include <asm/compat.h> | 32 | #include <asm/compat.h> |
33 | #include <asm/smp.h> | 33 | #include <asm/smp.h> |
34 | #include <asm/alternative.h> | ||
34 | 35 | ||
35 | #if 0 | 36 | #if 0 |
36 | #undef wrmsrl | 37 | #undef wrmsrl |
@@ -363,12 +364,18 @@ again: | |||
363 | return new_raw_count; | 364 | return new_raw_count; |
364 | } | 365 | } |
365 | 366 | ||
366 | /* using X86_FEATURE_PERFCTR_CORE to later implement ALTERNATIVE() here */ | ||
367 | static inline int x86_pmu_addr_offset(int index) | 367 | static inline int x86_pmu_addr_offset(int index) |
368 | { | 368 | { |
369 | if (boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) | 369 | int offset; |
370 | return index << 1; | 370 | |
371 | return index; | 371 | /* offset = X86_FEATURE_PERFCTR_CORE ? index << 1 : index */ |
372 | alternative_io(ASM_NOP2, | ||
373 | "shll $1, %%eax", | ||
374 | X86_FEATURE_PERFCTR_CORE, | ||
375 | "=a" (offset), | ||
376 | "a" (index)); | ||
377 | |||
378 | return offset; | ||
372 | } | 379 | } |
373 | 380 | ||
374 | static inline unsigned int x86_pmu_config_addr(int index) | 381 | static inline unsigned int x86_pmu_config_addr(int index) |
@@ -1766,17 +1773,6 @@ static struct pmu pmu = { | |||
1766 | * callchain support | 1773 | * callchain support |
1767 | */ | 1774 | */ |
1768 | 1775 | ||
1769 | static void | ||
1770 | backtrace_warning_symbol(void *data, char *msg, unsigned long symbol) | ||
1771 | { | ||
1772 | /* Ignore warnings */ | ||
1773 | } | ||
1774 | |||
1775 | static void backtrace_warning(void *data, char *msg) | ||
1776 | { | ||
1777 | /* Ignore warnings */ | ||
1778 | } | ||
1779 | |||
1780 | static int backtrace_stack(void *data, char *name) | 1776 | static int backtrace_stack(void *data, char *name) |
1781 | { | 1777 | { |
1782 | return 0; | 1778 | return 0; |
@@ -1790,8 +1786,6 @@ static void backtrace_address(void *data, unsigned long addr, int reliable) | |||
1790 | } | 1786 | } |
1791 | 1787 | ||
1792 | static const struct stacktrace_ops backtrace_ops = { | 1788 | static const struct stacktrace_ops backtrace_ops = { |
1793 | .warning = backtrace_warning, | ||
1794 | .warning_symbol = backtrace_warning_symbol, | ||
1795 | .stack = backtrace_stack, | 1789 | .stack = backtrace_stack, |
1796 | .address = backtrace_address, | 1790 | .address = backtrace_address, |
1797 | .walk_stack = print_context_stack_bp, | 1791 | .walk_stack = print_context_stack_bp, |
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c index cf4e369cea67..fe29c1d2219e 100644 --- a/arch/x86/kernel/cpu/perf_event_amd.c +++ b/arch/x86/kernel/cpu/perf_event_amd.c | |||
@@ -96,12 +96,14 @@ static __initconst const u64 amd_hw_cache_event_ids | |||
96 | */ | 96 | */ |
97 | static const u64 amd_perfmon_event_map[] = | 97 | static const u64 amd_perfmon_event_map[] = |
98 | { | 98 | { |
99 | [PERF_COUNT_HW_CPU_CYCLES] = 0x0076, | 99 | [PERF_COUNT_HW_CPU_CYCLES] = 0x0076, |
100 | [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, | 100 | [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, |
101 | [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080, | 101 | [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080, |
102 | [PERF_COUNT_HW_CACHE_MISSES] = 0x0081, | 102 | [PERF_COUNT_HW_CACHE_MISSES] = 0x0081, |
103 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2, | 103 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2, |
104 | [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3, | 104 | [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3, |
105 | [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */ | ||
106 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x00d1, /* "Dispatch stalls" event */ | ||
105 | }; | 107 | }; |
106 | 108 | ||
107 | static u64 amd_pmu_event_map(int hw_event) | 109 | static u64 amd_pmu_event_map(int hw_event) |
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index 447a28de6f09..41178c826c48 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c | |||
@@ -36,7 +36,7 @@ static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly = | |||
36 | [PERF_COUNT_HW_BUS_CYCLES] = 0x013c, | 36 | [PERF_COUNT_HW_BUS_CYCLES] = 0x013c, |
37 | }; | 37 | }; |
38 | 38 | ||
39 | static struct event_constraint intel_core_event_constraints[] = | 39 | static struct event_constraint intel_core_event_constraints[] __read_mostly = |
40 | { | 40 | { |
41 | INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */ | 41 | INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */ |
42 | INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */ | 42 | INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */ |
@@ -47,7 +47,7 @@ static struct event_constraint intel_core_event_constraints[] = | |||
47 | EVENT_CONSTRAINT_END | 47 | EVENT_CONSTRAINT_END |
48 | }; | 48 | }; |
49 | 49 | ||
50 | static struct event_constraint intel_core2_event_constraints[] = | 50 | static struct event_constraint intel_core2_event_constraints[] __read_mostly = |
51 | { | 51 | { |
52 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ | 52 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
53 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | 53 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ |
@@ -70,7 +70,7 @@ static struct event_constraint intel_core2_event_constraints[] = | |||
70 | EVENT_CONSTRAINT_END | 70 | EVENT_CONSTRAINT_END |
71 | }; | 71 | }; |
72 | 72 | ||
73 | static struct event_constraint intel_nehalem_event_constraints[] = | 73 | static struct event_constraint intel_nehalem_event_constraints[] __read_mostly = |
74 | { | 74 | { |
75 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ | 75 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
76 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | 76 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ |
@@ -86,19 +86,19 @@ static struct event_constraint intel_nehalem_event_constraints[] = | |||
86 | EVENT_CONSTRAINT_END | 86 | EVENT_CONSTRAINT_END |
87 | }; | 87 | }; |
88 | 88 | ||
89 | static struct extra_reg intel_nehalem_extra_regs[] = | 89 | static struct extra_reg intel_nehalem_extra_regs[] __read_mostly = |
90 | { | 90 | { |
91 | INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff), | 91 | INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff), |
92 | EVENT_EXTRA_END | 92 | EVENT_EXTRA_END |
93 | }; | 93 | }; |
94 | 94 | ||
95 | static struct event_constraint intel_nehalem_percore_constraints[] = | 95 | static struct event_constraint intel_nehalem_percore_constraints[] __read_mostly = |
96 | { | 96 | { |
97 | INTEL_EVENT_CONSTRAINT(0xb7, 0), | 97 | INTEL_EVENT_CONSTRAINT(0xb7, 0), |
98 | EVENT_CONSTRAINT_END | 98 | EVENT_CONSTRAINT_END |
99 | }; | 99 | }; |
100 | 100 | ||
101 | static struct event_constraint intel_westmere_event_constraints[] = | 101 | static struct event_constraint intel_westmere_event_constraints[] __read_mostly = |
102 | { | 102 | { |
103 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ | 103 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
104 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | 104 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ |
@@ -110,7 +110,7 @@ static struct event_constraint intel_westmere_event_constraints[] = | |||
110 | EVENT_CONSTRAINT_END | 110 | EVENT_CONSTRAINT_END |
111 | }; | 111 | }; |
112 | 112 | ||
113 | static struct event_constraint intel_snb_event_constraints[] = | 113 | static struct event_constraint intel_snb_event_constraints[] __read_mostly = |
114 | { | 114 | { |
115 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ | 115 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
116 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | 116 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ |
@@ -123,21 +123,21 @@ static struct event_constraint intel_snb_event_constraints[] = | |||
123 | EVENT_CONSTRAINT_END | 123 | EVENT_CONSTRAINT_END |
124 | }; | 124 | }; |
125 | 125 | ||
126 | static struct extra_reg intel_westmere_extra_regs[] = | 126 | static struct extra_reg intel_westmere_extra_regs[] __read_mostly = |
127 | { | 127 | { |
128 | INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff), | 128 | INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff), |
129 | INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff), | 129 | INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff), |
130 | EVENT_EXTRA_END | 130 | EVENT_EXTRA_END |
131 | }; | 131 | }; |
132 | 132 | ||
133 | static struct event_constraint intel_westmere_percore_constraints[] = | 133 | static struct event_constraint intel_westmere_percore_constraints[] __read_mostly = |
134 | { | 134 | { |
135 | INTEL_EVENT_CONSTRAINT(0xb7, 0), | 135 | INTEL_EVENT_CONSTRAINT(0xb7, 0), |
136 | INTEL_EVENT_CONSTRAINT(0xbb, 0), | 136 | INTEL_EVENT_CONSTRAINT(0xbb, 0), |
137 | EVENT_CONSTRAINT_END | 137 | EVENT_CONSTRAINT_END |
138 | }; | 138 | }; |
139 | 139 | ||
140 | static struct event_constraint intel_gen_event_constraints[] = | 140 | static struct event_constraint intel_gen_event_constraints[] __read_mostly = |
141 | { | 141 | { |
142 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ | 142 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
143 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | 143 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ |
@@ -1440,6 +1440,11 @@ static __init int intel_pmu_init(void) | |||
1440 | x86_pmu.enable_all = intel_pmu_nhm_enable_all; | 1440 | x86_pmu.enable_all = intel_pmu_nhm_enable_all; |
1441 | x86_pmu.extra_regs = intel_nehalem_extra_regs; | 1441 | x86_pmu.extra_regs = intel_nehalem_extra_regs; |
1442 | 1442 | ||
1443 | /* UOPS_ISSUED.STALLED_CYCLES */ | ||
1444 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; | ||
1445 | /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */ | ||
1446 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1; | ||
1447 | |||
1443 | if (ebx & 0x40) { | 1448 | if (ebx & 0x40) { |
1444 | /* | 1449 | /* |
1445 | * Erratum AAJ80 detected, we work it around by using | 1450 | * Erratum AAJ80 detected, we work it around by using |
@@ -1480,6 +1485,12 @@ static __init int intel_pmu_init(void) | |||
1480 | x86_pmu.enable_all = intel_pmu_nhm_enable_all; | 1485 | x86_pmu.enable_all = intel_pmu_nhm_enable_all; |
1481 | x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints; | 1486 | x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints; |
1482 | x86_pmu.extra_regs = intel_westmere_extra_regs; | 1487 | x86_pmu.extra_regs = intel_westmere_extra_regs; |
1488 | |||
1489 | /* UOPS_ISSUED.STALLED_CYCLES */ | ||
1490 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; | ||
1491 | /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */ | ||
1492 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1; | ||
1493 | |||
1483 | pr_cont("Westmere events, "); | 1494 | pr_cont("Westmere events, "); |
1484 | break; | 1495 | break; |
1485 | 1496 | ||
@@ -1491,6 +1502,12 @@ static __init int intel_pmu_init(void) | |||
1491 | 1502 | ||
1492 | x86_pmu.event_constraints = intel_snb_event_constraints; | 1503 | x86_pmu.event_constraints = intel_snb_event_constraints; |
1493 | x86_pmu.pebs_constraints = intel_snb_pebs_events; | 1504 | x86_pmu.pebs_constraints = intel_snb_pebs_events; |
1505 | |||
1506 | /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ | ||
1507 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; | ||
1508 | /* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/ | ||
1509 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x18001b1; | ||
1510 | |||
1494 | pr_cont("SandyBridge events, "); | 1511 | pr_cont("SandyBridge events, "); |
1495 | break; | 1512 | break; |
1496 | 1513 | ||
diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c index e93fcd55fae1..ead584fb6a7d 100644 --- a/arch/x86/kernel/cpu/perf_event_p4.c +++ b/arch/x86/kernel/cpu/perf_event_p4.c | |||
@@ -468,7 +468,7 @@ static struct p4_event_bind p4_event_bind_map[] = { | |||
468 | .opcode = P4_OPCODE(P4_EVENT_MISPRED_BRANCH_RETIRED), | 468 | .opcode = P4_OPCODE(P4_EVENT_MISPRED_BRANCH_RETIRED), |
469 | .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 }, | 469 | .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 }, |
470 | .escr_emask = | 470 | .escr_emask = |
471 | P4_ESCR_EMASK_BIT(P4_EVENT_MISPRED_BRANCH_RETIRED, NBOGUS), | 471 | P4_ESCR_EMASK_BIT(P4_EVENT_MISPRED_BRANCH_RETIRED, NBOGUS), |
472 | .cntr = { {12, 13, 16}, {14, 15, 17} }, | 472 | .cntr = { {12, 13, 16}, {14, 15, 17} }, |
473 | }, | 473 | }, |
474 | [P4_EVENT_X87_ASSIST] = { | 474 | [P4_EVENT_X87_ASSIST] = { |
@@ -912,8 +912,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) | |||
912 | int idx, handled = 0; | 912 | int idx, handled = 0; |
913 | u64 val; | 913 | u64 val; |
914 | 914 | ||
915 | data.addr = 0; | 915 | perf_sample_data_init(&data, 0); |
916 | data.raw = NULL; | ||
917 | 916 | ||
918 | cpuc = &__get_cpu_var(cpu_hw_events); | 917 | cpuc = &__get_cpu_var(cpu_hw_events); |
919 | 918 | ||
@@ -1197,7 +1196,7 @@ static __init int p4_pmu_init(void) | |||
1197 | { | 1196 | { |
1198 | unsigned int low, high; | 1197 | unsigned int low, high; |
1199 | 1198 | ||
1200 | /* If we get stripped -- indexig fails */ | 1199 | /* If we get stripped -- indexing fails */ |
1201 | BUILD_BUG_ON(ARCH_P4_MAX_CCCR > X86_PMC_MAX_GENERIC); | 1200 | BUILD_BUG_ON(ARCH_P4_MAX_CCCR > X86_PMC_MAX_GENERIC); |
1202 | 1201 | ||
1203 | rdmsr(MSR_IA32_MISC_ENABLE, low, high); | 1202 | rdmsr(MSR_IA32_MISC_ENABLE, low, high); |
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index e2a3f0606da4..f478ff6877ef 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c | |||
@@ -135,20 +135,6 @@ print_context_stack_bp(struct thread_info *tinfo, | |||
135 | } | 135 | } |
136 | EXPORT_SYMBOL_GPL(print_context_stack_bp); | 136 | EXPORT_SYMBOL_GPL(print_context_stack_bp); |
137 | 137 | ||
138 | |||
139 | static void | ||
140 | print_trace_warning_symbol(void *data, char *msg, unsigned long symbol) | ||
141 | { | ||
142 | printk(data); | ||
143 | print_symbol(msg, symbol); | ||
144 | printk("\n"); | ||
145 | } | ||
146 | |||
147 | static void print_trace_warning(void *data, char *msg) | ||
148 | { | ||
149 | printk("%s%s\n", (char *)data, msg); | ||
150 | } | ||
151 | |||
152 | static int print_trace_stack(void *data, char *name) | 138 | static int print_trace_stack(void *data, char *name) |
153 | { | 139 | { |
154 | printk("%s <%s> ", (char *)data, name); | 140 | printk("%s <%s> ", (char *)data, name); |
@@ -166,8 +152,6 @@ static void print_trace_address(void *data, unsigned long addr, int reliable) | |||
166 | } | 152 | } |
167 | 153 | ||
168 | static const struct stacktrace_ops print_trace_ops = { | 154 | static const struct stacktrace_ops print_trace_ops = { |
169 | .warning = print_trace_warning, | ||
170 | .warning_symbol = print_trace_warning_symbol, | ||
171 | .stack = print_trace_stack, | 155 | .stack = print_trace_stack, |
172 | .address = print_trace_address, | 156 | .address = print_trace_address, |
173 | .walk_stack = print_context_stack, | 157 | .walk_stack = print_context_stack, |
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index ab23f1ad4bf1..52f256f2cc81 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c | |||
@@ -24,6 +24,7 @@ | |||
24 | #include <linux/bug.h> | 24 | #include <linux/bug.h> |
25 | #include <linux/mm.h> | 25 | #include <linux/mm.h> |
26 | #include <linux/gfp.h> | 26 | #include <linux/gfp.h> |
27 | #include <linux/jump_label.h> | ||
27 | 28 | ||
28 | #include <asm/system.h> | 29 | #include <asm/system.h> |
29 | #include <asm/page.h> | 30 | #include <asm/page.h> |
diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c index 6515733a289d..55d9bc03f696 100644 --- a/arch/x86/kernel/stacktrace.c +++ b/arch/x86/kernel/stacktrace.c | |||
@@ -9,15 +9,6 @@ | |||
9 | #include <linux/uaccess.h> | 9 | #include <linux/uaccess.h> |
10 | #include <asm/stacktrace.h> | 10 | #include <asm/stacktrace.h> |
11 | 11 | ||
12 | static void save_stack_warning(void *data, char *msg) | ||
13 | { | ||
14 | } | ||
15 | |||
16 | static void | ||
17 | save_stack_warning_symbol(void *data, char *msg, unsigned long symbol) | ||
18 | { | ||
19 | } | ||
20 | |||
21 | static int save_stack_stack(void *data, char *name) | 12 | static int save_stack_stack(void *data, char *name) |
22 | { | 13 | { |
23 | return 0; | 14 | return 0; |
@@ -53,16 +44,12 @@ save_stack_address_nosched(void *data, unsigned long addr, int reliable) | |||
53 | } | 44 | } |
54 | 45 | ||
55 | static const struct stacktrace_ops save_stack_ops = { | 46 | static const struct stacktrace_ops save_stack_ops = { |
56 | .warning = save_stack_warning, | ||
57 | .warning_symbol = save_stack_warning_symbol, | ||
58 | .stack = save_stack_stack, | 47 | .stack = save_stack_stack, |
59 | .address = save_stack_address, | 48 | .address = save_stack_address, |
60 | .walk_stack = print_context_stack, | 49 | .walk_stack = print_context_stack, |
61 | }; | 50 | }; |
62 | 51 | ||
63 | static const struct stacktrace_ops save_stack_ops_nosched = { | 52 | static const struct stacktrace_ops save_stack_ops_nosched = { |
64 | .warning = save_stack_warning, | ||
65 | .warning_symbol = save_stack_warning_symbol, | ||
66 | .stack = save_stack_stack, | 53 | .stack = save_stack_stack, |
67 | .address = save_stack_address_nosched, | 54 | .address = save_stack_address_nosched, |
68 | .walk_stack = print_context_stack, | 55 | .walk_stack = print_context_stack, |
diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S index aa4326bfb24a..f2145cfa12a6 100644 --- a/arch/x86/lib/clear_page_64.S +++ b/arch/x86/lib/clear_page_64.S | |||
@@ -1,5 +1,6 @@ | |||
1 | #include <linux/linkage.h> | 1 | #include <linux/linkage.h> |
2 | #include <asm/dwarf2.h> | 2 | #include <asm/dwarf2.h> |
3 | #include <asm/alternative-asm.h> | ||
3 | 4 | ||
4 | /* | 5 | /* |
5 | * Zero a page. | 6 | * Zero a page. |
@@ -14,6 +15,15 @@ ENTRY(clear_page_c) | |||
14 | CFI_ENDPROC | 15 | CFI_ENDPROC |
15 | ENDPROC(clear_page_c) | 16 | ENDPROC(clear_page_c) |
16 | 17 | ||
18 | ENTRY(clear_page_c_e) | ||
19 | CFI_STARTPROC | ||
20 | movl $4096,%ecx | ||
21 | xorl %eax,%eax | ||
22 | rep stosb | ||
23 | ret | ||
24 | CFI_ENDPROC | ||
25 | ENDPROC(clear_page_c_e) | ||
26 | |||
17 | ENTRY(clear_page) | 27 | ENTRY(clear_page) |
18 | CFI_STARTPROC | 28 | CFI_STARTPROC |
19 | xorl %eax,%eax | 29 | xorl %eax,%eax |
@@ -38,21 +48,26 @@ ENTRY(clear_page) | |||
38 | .Lclear_page_end: | 48 | .Lclear_page_end: |
39 | ENDPROC(clear_page) | 49 | ENDPROC(clear_page) |
40 | 50 | ||
41 | /* Some CPUs run faster using the string instructions. | 51 | /* |
42 | It is also a lot simpler. Use this when possible */ | 52 | * Some CPUs support enhanced REP MOVSB/STOSB instructions. |
53 | * It is recommended to use this when possible. | ||
54 | * If enhanced REP MOVSB/STOSB is not available, try to use fast string. | ||
55 | * Otherwise, use original function. | ||
56 | * | ||
57 | */ | ||
43 | 58 | ||
44 | #include <asm/cpufeature.h> | 59 | #include <asm/cpufeature.h> |
45 | 60 | ||
46 | .section .altinstr_replacement,"ax" | 61 | .section .altinstr_replacement,"ax" |
47 | 1: .byte 0xeb /* jmp <disp8> */ | 62 | 1: .byte 0xeb /* jmp <disp8> */ |
48 | .byte (clear_page_c - clear_page) - (2f - 1b) /* offset */ | 63 | .byte (clear_page_c - clear_page) - (2f - 1b) /* offset */ |
49 | 2: | 64 | 2: .byte 0xeb /* jmp <disp8> */ |
65 | .byte (clear_page_c_e - clear_page) - (3f - 2b) /* offset */ | ||
66 | 3: | ||
50 | .previous | 67 | .previous |
51 | .section .altinstructions,"a" | 68 | .section .altinstructions,"a" |
52 | .align 8 | 69 | altinstruction_entry clear_page,1b,X86_FEATURE_REP_GOOD,\ |
53 | .quad clear_page | 70 | .Lclear_page_end-clear_page, 2b-1b |
54 | .quad 1b | 71 | altinstruction_entry clear_page,2b,X86_FEATURE_ERMS, \ |
55 | .word X86_FEATURE_REP_GOOD | 72 | .Lclear_page_end-clear_page,3b-2b |
56 | .byte .Lclear_page_end - clear_page | ||
57 | .byte 2b - 1b | ||
58 | .previous | 73 | .previous |
diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S index 99e482615195..024840266ba0 100644 --- a/arch/x86/lib/copy_user_64.S +++ b/arch/x86/lib/copy_user_64.S | |||
@@ -15,23 +15,30 @@ | |||
15 | #include <asm/asm-offsets.h> | 15 | #include <asm/asm-offsets.h> |
16 | #include <asm/thread_info.h> | 16 | #include <asm/thread_info.h> |
17 | #include <asm/cpufeature.h> | 17 | #include <asm/cpufeature.h> |
18 | #include <asm/alternative-asm.h> | ||
18 | 19 | ||
19 | .macro ALTERNATIVE_JUMP feature,orig,alt | 20 | /* |
21 | * By placing feature2 after feature1 in altinstructions section, we logically | ||
22 | * implement: | ||
23 | * If CPU has feature2, jmp to alt2 is used | ||
24 | * else if CPU has feature1, jmp to alt1 is used | ||
25 | * else jmp to orig is used. | ||
26 | */ | ||
27 | .macro ALTERNATIVE_JUMP feature1,feature2,orig,alt1,alt2 | ||
20 | 0: | 28 | 0: |
21 | .byte 0xe9 /* 32bit jump */ | 29 | .byte 0xe9 /* 32bit jump */ |
22 | .long \orig-1f /* by default jump to orig */ | 30 | .long \orig-1f /* by default jump to orig */ |
23 | 1: | 31 | 1: |
24 | .section .altinstr_replacement,"ax" | 32 | .section .altinstr_replacement,"ax" |
25 | 2: .byte 0xe9 /* near jump with 32bit immediate */ | 33 | 2: .byte 0xe9 /* near jump with 32bit immediate */ |
26 | .long \alt-1b /* offset */ /* or alternatively to alt */ | 34 | .long \alt1-1b /* offset */ /* or alternatively to alt1 */ |
35 | 3: .byte 0xe9 /* near jump with 32bit immediate */ | ||
36 | .long \alt2-1b /* offset */ /* or alternatively to alt2 */ | ||
27 | .previous | 37 | .previous |
38 | |||
28 | .section .altinstructions,"a" | 39 | .section .altinstructions,"a" |
29 | .align 8 | 40 | altinstruction_entry 0b,2b,\feature1,5,5 |
30 | .quad 0b | 41 | altinstruction_entry 0b,3b,\feature2,5,5 |
31 | .quad 2b | ||
32 | .word \feature /* when feature is set */ | ||
33 | .byte 5 | ||
34 | .byte 5 | ||
35 | .previous | 42 | .previous |
36 | .endm | 43 | .endm |
37 | 44 | ||
@@ -72,8 +79,10 @@ ENTRY(_copy_to_user) | |||
72 | addq %rdx,%rcx | 79 | addq %rdx,%rcx |
73 | jc bad_to_user | 80 | jc bad_to_user |
74 | cmpq TI_addr_limit(%rax),%rcx | 81 | cmpq TI_addr_limit(%rax),%rcx |
75 | jae bad_to_user | 82 | ja bad_to_user |
76 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string | 83 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \ |
84 | copy_user_generic_unrolled,copy_user_generic_string, \ | ||
85 | copy_user_enhanced_fast_string | ||
77 | CFI_ENDPROC | 86 | CFI_ENDPROC |
78 | ENDPROC(_copy_to_user) | 87 | ENDPROC(_copy_to_user) |
79 | 88 | ||
@@ -85,8 +94,10 @@ ENTRY(_copy_from_user) | |||
85 | addq %rdx,%rcx | 94 | addq %rdx,%rcx |
86 | jc bad_from_user | 95 | jc bad_from_user |
87 | cmpq TI_addr_limit(%rax),%rcx | 96 | cmpq TI_addr_limit(%rax),%rcx |
88 | jae bad_from_user | 97 | ja bad_from_user |
89 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string | 98 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \ |
99 | copy_user_generic_unrolled,copy_user_generic_string, \ | ||
100 | copy_user_enhanced_fast_string | ||
90 | CFI_ENDPROC | 101 | CFI_ENDPROC |
91 | ENDPROC(_copy_from_user) | 102 | ENDPROC(_copy_from_user) |
92 | 103 | ||
@@ -255,3 +266,37 @@ ENTRY(copy_user_generic_string) | |||
255 | .previous | 266 | .previous |
256 | CFI_ENDPROC | 267 | CFI_ENDPROC |
257 | ENDPROC(copy_user_generic_string) | 268 | ENDPROC(copy_user_generic_string) |
269 | |||
270 | /* | ||
271 | * Some CPUs are adding enhanced REP MOVSB/STOSB instructions. | ||
272 | * It's recommended to use enhanced REP MOVSB/STOSB if it's enabled. | ||
273 | * | ||
274 | * Input: | ||
275 | * rdi destination | ||
276 | * rsi source | ||
277 | * rdx count | ||
278 | * | ||
279 | * Output: | ||
280 | * eax uncopied bytes or 0 if successful. | ||
281 | */ | ||
282 | ENTRY(copy_user_enhanced_fast_string) | ||
283 | CFI_STARTPROC | ||
284 | andl %edx,%edx | ||
285 | jz 2f | ||
286 | movl %edx,%ecx | ||
287 | 1: rep | ||
288 | movsb | ||
289 | 2: xorl %eax,%eax | ||
290 | ret | ||
291 | |||
292 | .section .fixup,"ax" | ||
293 | 12: movl %ecx,%edx /* ecx is zerorest also */ | ||
294 | jmp copy_user_handle_tail | ||
295 | .previous | ||
296 | |||
297 | .section __ex_table,"a" | ||
298 | .align 8 | ||
299 | .quad 1b,12b | ||
300 | .previous | ||
301 | CFI_ENDPROC | ||
302 | ENDPROC(copy_user_enhanced_fast_string) | ||
diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S index 75ef61e35e38..daab21dae2d1 100644 --- a/arch/x86/lib/memcpy_64.S +++ b/arch/x86/lib/memcpy_64.S | |||
@@ -4,6 +4,7 @@ | |||
4 | 4 | ||
5 | #include <asm/cpufeature.h> | 5 | #include <asm/cpufeature.h> |
6 | #include <asm/dwarf2.h> | 6 | #include <asm/dwarf2.h> |
7 | #include <asm/alternative-asm.h> | ||
7 | 8 | ||
8 | /* | 9 | /* |
9 | * memcpy - Copy a memory block. | 10 | * memcpy - Copy a memory block. |
@@ -37,6 +38,23 @@ | |||
37 | .Lmemcpy_e: | 38 | .Lmemcpy_e: |
38 | .previous | 39 | .previous |
39 | 40 | ||
41 | /* | ||
42 | * memcpy_c_e() - enhanced fast string memcpy. This is faster and simpler than | ||
43 | * memcpy_c. Use memcpy_c_e when possible. | ||
44 | * | ||
45 | * This gets patched over the unrolled variant (below) via the | ||
46 | * alternative instructions framework: | ||
47 | */ | ||
48 | .section .altinstr_replacement, "ax", @progbits | ||
49 | .Lmemcpy_c_e: | ||
50 | movq %rdi, %rax | ||
51 | |||
52 | movl %edx, %ecx | ||
53 | rep movsb | ||
54 | ret | ||
55 | .Lmemcpy_e_e: | ||
56 | .previous | ||
57 | |||
40 | ENTRY(__memcpy) | 58 | ENTRY(__memcpy) |
41 | ENTRY(memcpy) | 59 | ENTRY(memcpy) |
42 | CFI_STARTPROC | 60 | CFI_STARTPROC |
@@ -171,21 +189,22 @@ ENDPROC(memcpy) | |||
171 | ENDPROC(__memcpy) | 189 | ENDPROC(__memcpy) |
172 | 190 | ||
173 | /* | 191 | /* |
174 | * Some CPUs run faster using the string copy instructions. | 192 | * Some CPUs are adding enhanced REP MOVSB/STOSB feature |
175 | * It is also a lot simpler. Use this when possible: | 193 | * If the feature is supported, memcpy_c_e() is the first choice. |
176 | */ | 194 | * If enhanced rep movsb copy is not available, use fast string copy |
177 | 195 | * memcpy_c() when possible. This is faster and code is simpler than | |
178 | .section .altinstructions, "a" | 196 | * original memcpy(). |
179 | .align 8 | 197 | * Otherwise, original memcpy() is used. |
180 | .quad memcpy | 198 | * In .altinstructions section, ERMS feature is placed after REG_GOOD |
181 | .quad .Lmemcpy_c | 199 | * feature to implement the right patch order. |
182 | .word X86_FEATURE_REP_GOOD | 200 | * |
183 | |||
184 | /* | ||
185 | * Replace only beginning, memcpy is used to apply alternatives, | 201 | * Replace only beginning, memcpy is used to apply alternatives, |
186 | * so it is silly to overwrite itself with nops - reboot is the | 202 | * so it is silly to overwrite itself with nops - reboot is the |
187 | * only outcome... | 203 | * only outcome... |
188 | */ | 204 | */ |
189 | .byte .Lmemcpy_e - .Lmemcpy_c | 205 | .section .altinstructions, "a" |
190 | .byte .Lmemcpy_e - .Lmemcpy_c | 206 | altinstruction_entry memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\ |
207 | .Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c | ||
208 | altinstruction_entry memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \ | ||
209 | .Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e | ||
191 | .previous | 210 | .previous |
diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S index 0ecb8433e5a8..d0ec9c2936d7 100644 --- a/arch/x86/lib/memmove_64.S +++ b/arch/x86/lib/memmove_64.S | |||
@@ -8,6 +8,7 @@ | |||
8 | #define _STRING_C | 8 | #define _STRING_C |
9 | #include <linux/linkage.h> | 9 | #include <linux/linkage.h> |
10 | #include <asm/dwarf2.h> | 10 | #include <asm/dwarf2.h> |
11 | #include <asm/cpufeature.h> | ||
11 | 12 | ||
12 | #undef memmove | 13 | #undef memmove |
13 | 14 | ||
@@ -24,6 +25,7 @@ | |||
24 | */ | 25 | */ |
25 | ENTRY(memmove) | 26 | ENTRY(memmove) |
26 | CFI_STARTPROC | 27 | CFI_STARTPROC |
28 | |||
27 | /* Handle more 32bytes in loop */ | 29 | /* Handle more 32bytes in loop */ |
28 | mov %rdi, %rax | 30 | mov %rdi, %rax |
29 | cmp $0x20, %rdx | 31 | cmp $0x20, %rdx |
@@ -31,8 +33,13 @@ ENTRY(memmove) | |||
31 | 33 | ||
32 | /* Decide forward/backward copy mode */ | 34 | /* Decide forward/backward copy mode */ |
33 | cmp %rdi, %rsi | 35 | cmp %rdi, %rsi |
34 | jb 2f | 36 | jge .Lmemmove_begin_forward |
37 | mov %rsi, %r8 | ||
38 | add %rdx, %r8 | ||
39 | cmp %rdi, %r8 | ||
40 | jg 2f | ||
35 | 41 | ||
42 | .Lmemmove_begin_forward: | ||
36 | /* | 43 | /* |
37 | * movsq instruction have many startup latency | 44 | * movsq instruction have many startup latency |
38 | * so we handle small size by general register. | 45 | * so we handle small size by general register. |
@@ -78,6 +85,8 @@ ENTRY(memmove) | |||
78 | rep movsq | 85 | rep movsq |
79 | movq %r11, (%r10) | 86 | movq %r11, (%r10) |
80 | jmp 13f | 87 | jmp 13f |
88 | .Lmemmove_end_forward: | ||
89 | |||
81 | /* | 90 | /* |
82 | * Handle data backward by movsq. | 91 | * Handle data backward by movsq. |
83 | */ | 92 | */ |
@@ -194,4 +203,22 @@ ENTRY(memmove) | |||
194 | 13: | 203 | 13: |
195 | retq | 204 | retq |
196 | CFI_ENDPROC | 205 | CFI_ENDPROC |
206 | |||
207 | .section .altinstr_replacement,"ax" | ||
208 | .Lmemmove_begin_forward_efs: | ||
209 | /* Forward moving data. */ | ||
210 | movq %rdx, %rcx | ||
211 | rep movsb | ||
212 | retq | ||
213 | .Lmemmove_end_forward_efs: | ||
214 | .previous | ||
215 | |||
216 | .section .altinstructions,"a" | ||
217 | .align 8 | ||
218 | .quad .Lmemmove_begin_forward | ||
219 | .quad .Lmemmove_begin_forward_efs | ||
220 | .word X86_FEATURE_ERMS | ||
221 | .byte .Lmemmove_end_forward-.Lmemmove_begin_forward | ||
222 | .byte .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs | ||
223 | .previous | ||
197 | ENDPROC(memmove) | 224 | ENDPROC(memmove) |
diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S index 09d344269652..79bd454b78a3 100644 --- a/arch/x86/lib/memset_64.S +++ b/arch/x86/lib/memset_64.S | |||
@@ -2,9 +2,13 @@ | |||
2 | 2 | ||
3 | #include <linux/linkage.h> | 3 | #include <linux/linkage.h> |
4 | #include <asm/dwarf2.h> | 4 | #include <asm/dwarf2.h> |
5 | #include <asm/cpufeature.h> | ||
6 | #include <asm/alternative-asm.h> | ||
5 | 7 | ||
6 | /* | 8 | /* |
7 | * ISO C memset - set a memory block to a byte value. | 9 | * ISO C memset - set a memory block to a byte value. This function uses fast |
10 | * string to get better performance than the original function. The code is | ||
11 | * simpler and shorter than the orignal function as well. | ||
8 | * | 12 | * |
9 | * rdi destination | 13 | * rdi destination |
10 | * rsi value (char) | 14 | * rsi value (char) |
@@ -31,6 +35,28 @@ | |||
31 | .Lmemset_e: | 35 | .Lmemset_e: |
32 | .previous | 36 | .previous |
33 | 37 | ||
38 | /* | ||
39 | * ISO C memset - set a memory block to a byte value. This function uses | ||
40 | * enhanced rep stosb to override the fast string function. | ||
41 | * The code is simpler and shorter than the fast string function as well. | ||
42 | * | ||
43 | * rdi destination | ||
44 | * rsi value (char) | ||
45 | * rdx count (bytes) | ||
46 | * | ||
47 | * rax original destination | ||
48 | */ | ||
49 | .section .altinstr_replacement, "ax", @progbits | ||
50 | .Lmemset_c_e: | ||
51 | movq %rdi,%r9 | ||
52 | movb %sil,%al | ||
53 | movl %edx,%ecx | ||
54 | rep stosb | ||
55 | movq %r9,%rax | ||
56 | ret | ||
57 | .Lmemset_e_e: | ||
58 | .previous | ||
59 | |||
34 | ENTRY(memset) | 60 | ENTRY(memset) |
35 | ENTRY(__memset) | 61 | ENTRY(__memset) |
36 | CFI_STARTPROC | 62 | CFI_STARTPROC |
@@ -112,16 +138,20 @@ ENTRY(__memset) | |||
112 | ENDPROC(memset) | 138 | ENDPROC(memset) |
113 | ENDPROC(__memset) | 139 | ENDPROC(__memset) |
114 | 140 | ||
115 | /* Some CPUs run faster using the string instructions. | 141 | /* Some CPUs support enhanced REP MOVSB/STOSB feature. |
116 | It is also a lot simpler. Use this when possible */ | 142 | * It is recommended to use this when possible. |
117 | 143 | * | |
118 | #include <asm/cpufeature.h> | 144 | * If enhanced REP MOVSB/STOSB feature is not available, use fast string |
119 | 145 | * instructions. | |
146 | * | ||
147 | * Otherwise, use original memset function. | ||
148 | * | ||
149 | * In .altinstructions section, ERMS feature is placed after REG_GOOD | ||
150 | * feature to implement the right patch order. | ||
151 | */ | ||
120 | .section .altinstructions,"a" | 152 | .section .altinstructions,"a" |
121 | .align 8 | 153 | altinstruction_entry memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\ |
122 | .quad memset | 154 | .Lfinal-memset,.Lmemset_e-.Lmemset_c |
123 | .quad .Lmemset_c | 155 | altinstruction_entry memset,.Lmemset_c_e,X86_FEATURE_ERMS, \ |
124 | .word X86_FEATURE_REP_GOOD | 156 | .Lfinal-memset,.Lmemset_e_e-.Lmemset_c_e |
125 | .byte .Lfinal - memset | ||
126 | .byte .Lmemset_e - .Lmemset_c | ||
127 | .previous | 157 | .previous |
diff --git a/arch/x86/oprofile/backtrace.c b/arch/x86/oprofile/backtrace.c index 2d49d4e19a36..a5b64ab4cd6e 100644 --- a/arch/x86/oprofile/backtrace.c +++ b/arch/x86/oprofile/backtrace.c | |||
@@ -16,17 +16,6 @@ | |||
16 | #include <asm/stacktrace.h> | 16 | #include <asm/stacktrace.h> |
17 | #include <linux/compat.h> | 17 | #include <linux/compat.h> |
18 | 18 | ||
19 | static void backtrace_warning_symbol(void *data, char *msg, | ||
20 | unsigned long symbol) | ||
21 | { | ||
22 | /* Ignore warnings */ | ||
23 | } | ||
24 | |||
25 | static void backtrace_warning(void *data, char *msg) | ||
26 | { | ||
27 | /* Ignore warnings */ | ||
28 | } | ||
29 | |||
30 | static int backtrace_stack(void *data, char *name) | 19 | static int backtrace_stack(void *data, char *name) |
31 | { | 20 | { |
32 | /* Yes, we want all stacks */ | 21 | /* Yes, we want all stacks */ |
@@ -42,8 +31,6 @@ static void backtrace_address(void *data, unsigned long addr, int reliable) | |||
42 | } | 31 | } |
43 | 32 | ||
44 | static struct stacktrace_ops backtrace_ops = { | 33 | static struct stacktrace_ops backtrace_ops = { |
45 | .warning = backtrace_warning, | ||
46 | .warning_symbol = backtrace_warning_symbol, | ||
47 | .stack = backtrace_stack, | 34 | .stack = backtrace_stack, |
48 | .address = backtrace_address, | 35 | .address = backtrace_address, |
49 | .walk_stack = print_context_stack, | 36 | .walk_stack = print_context_stack, |
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index b27445e00b6c..077c00d94f6e 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h | |||
@@ -170,6 +170,10 @@ | |||
170 | STRUCT_ALIGN(); \ | 170 | STRUCT_ALIGN(); \ |
171 | *(__tracepoints) \ | 171 | *(__tracepoints) \ |
172 | /* implement dynamic printk debug */ \ | 172 | /* implement dynamic printk debug */ \ |
173 | . = ALIGN(8); \ | ||
174 | VMLINUX_SYMBOL(__start___jump_table) = .; \ | ||
175 | *(__jump_table) \ | ||
176 | VMLINUX_SYMBOL(__stop___jump_table) = .; \ | ||
173 | . = ALIGN(8); \ | 177 | . = ALIGN(8); \ |
174 | VMLINUX_SYMBOL(__start___verbose) = .; \ | 178 | VMLINUX_SYMBOL(__start___verbose) = .; \ |
175 | *(__verbose) \ | 179 | *(__verbose) \ |
@@ -228,8 +232,6 @@ | |||
228 | \ | 232 | \ |
229 | BUG_TABLE \ | 233 | BUG_TABLE \ |
230 | \ | 234 | \ |
231 | JUMP_TABLE \ | ||
232 | \ | ||
233 | /* PCI quirks */ \ | 235 | /* PCI quirks */ \ |
234 | .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ | 236 | .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ |
235 | VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \ | 237 | VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \ |
@@ -589,14 +591,6 @@ | |||
589 | #define BUG_TABLE | 591 | #define BUG_TABLE |
590 | #endif | 592 | #endif |
591 | 593 | ||
592 | #define JUMP_TABLE \ | ||
593 | . = ALIGN(8); \ | ||
594 | __jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) { \ | ||
595 | VMLINUX_SYMBOL(__start___jump_table) = .; \ | ||
596 | *(__jump_table) \ | ||
597 | VMLINUX_SYMBOL(__stop___jump_table) = .; \ | ||
598 | } | ||
599 | |||
600 | #ifdef CONFIG_PM_TRACE | 594 | #ifdef CONFIG_PM_TRACE |
601 | #define TRACEDATA \ | 595 | #define TRACEDATA \ |
602 | . = ALIGN(4); \ | 596 | . = ALIGN(4); \ |
diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h index 0c9653f11c18..e747ecd48e1c 100644 --- a/include/linux/dynamic_debug.h +++ b/include/linux/dynamic_debug.h | |||
@@ -1,8 +1,6 @@ | |||
1 | #ifndef _DYNAMIC_DEBUG_H | 1 | #ifndef _DYNAMIC_DEBUG_H |
2 | #define _DYNAMIC_DEBUG_H | 2 | #define _DYNAMIC_DEBUG_H |
3 | 3 | ||
4 | #include <linux/jump_label.h> | ||
5 | |||
6 | /* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which | 4 | /* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which |
7 | * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They | 5 | * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They |
8 | * use independent hash functions, to reduce the chance of false positives. | 6 | * use independent hash functions, to reduce the chance of false positives. |
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index ca29e03c1fac..9d88e1cb5dbb 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h | |||
@@ -29,9 +29,22 @@ ftrace_enable_sysctl(struct ctl_table *table, int write, | |||
29 | 29 | ||
30 | typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip); | 30 | typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip); |
31 | 31 | ||
32 | struct ftrace_hash; | ||
33 | |||
34 | enum { | ||
35 | FTRACE_OPS_FL_ENABLED = 1 << 0, | ||
36 | FTRACE_OPS_FL_GLOBAL = 1 << 1, | ||
37 | FTRACE_OPS_FL_DYNAMIC = 1 << 2, | ||
38 | }; | ||
39 | |||
32 | struct ftrace_ops { | 40 | struct ftrace_ops { |
33 | ftrace_func_t func; | 41 | ftrace_func_t func; |
34 | struct ftrace_ops *next; | 42 | struct ftrace_ops *next; |
43 | unsigned long flags; | ||
44 | #ifdef CONFIG_DYNAMIC_FTRACE | ||
45 | struct ftrace_hash *notrace_hash; | ||
46 | struct ftrace_hash *filter_hash; | ||
47 | #endif | ||
35 | }; | 48 | }; |
36 | 49 | ||
37 | extern int function_trace_stop; | 50 | extern int function_trace_stop; |
@@ -146,14 +159,13 @@ extern void unregister_ftrace_function_probe_all(char *glob); | |||
146 | extern int ftrace_text_reserved(void *start, void *end); | 159 | extern int ftrace_text_reserved(void *start, void *end); |
147 | 160 | ||
148 | enum { | 161 | enum { |
149 | FTRACE_FL_FREE = (1 << 0), | 162 | FTRACE_FL_ENABLED = (1 << 30), |
150 | FTRACE_FL_FAILED = (1 << 1), | 163 | FTRACE_FL_FREE = (1 << 31), |
151 | FTRACE_FL_FILTER = (1 << 2), | ||
152 | FTRACE_FL_ENABLED = (1 << 3), | ||
153 | FTRACE_FL_NOTRACE = (1 << 4), | ||
154 | FTRACE_FL_CONVERTED = (1 << 5), | ||
155 | }; | 164 | }; |
156 | 165 | ||
166 | #define FTRACE_FL_MASK (0x3UL << 30) | ||
167 | #define FTRACE_REF_MAX ((1 << 30) - 1) | ||
168 | |||
157 | struct dyn_ftrace { | 169 | struct dyn_ftrace { |
158 | union { | 170 | union { |
159 | unsigned long ip; /* address of mcount call-site */ | 171 | unsigned long ip; /* address of mcount call-site */ |
@@ -167,7 +179,12 @@ struct dyn_ftrace { | |||
167 | }; | 179 | }; |
168 | 180 | ||
169 | int ftrace_force_update(void); | 181 | int ftrace_force_update(void); |
170 | void ftrace_set_filter(unsigned char *buf, int len, int reset); | 182 | void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, |
183 | int len, int reset); | ||
184 | void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, | ||
185 | int len, int reset); | ||
186 | void ftrace_set_global_filter(unsigned char *buf, int len, int reset); | ||
187 | void ftrace_set_global_notrace(unsigned char *buf, int len, int reset); | ||
171 | 188 | ||
172 | int register_ftrace_command(struct ftrace_func_command *cmd); | 189 | int register_ftrace_command(struct ftrace_func_command *cmd); |
173 | int unregister_ftrace_command(struct ftrace_func_command *cmd); | 190 | int unregister_ftrace_command(struct ftrace_func_command *cmd); |
diff --git a/include/linux/init.h b/include/linux/init.h index 577671c55153..9146f39cdddf 100644 --- a/include/linux/init.h +++ b/include/linux/init.h | |||
@@ -79,29 +79,29 @@ | |||
79 | #define __exitused __used | 79 | #define __exitused __used |
80 | #endif | 80 | #endif |
81 | 81 | ||
82 | #define __exit __section(.exit.text) __exitused __cold | 82 | #define __exit __section(.exit.text) __exitused __cold notrace |
83 | 83 | ||
84 | /* Used for HOTPLUG */ | 84 | /* Used for HOTPLUG */ |
85 | #define __devinit __section(.devinit.text) __cold | 85 | #define __devinit __section(.devinit.text) __cold notrace |
86 | #define __devinitdata __section(.devinit.data) | 86 | #define __devinitdata __section(.devinit.data) |
87 | #define __devinitconst __section(.devinit.rodata) | 87 | #define __devinitconst __section(.devinit.rodata) |
88 | #define __devexit __section(.devexit.text) __exitused __cold | 88 | #define __devexit __section(.devexit.text) __exitused __cold notrace |
89 | #define __devexitdata __section(.devexit.data) | 89 | #define __devexitdata __section(.devexit.data) |
90 | #define __devexitconst __section(.devexit.rodata) | 90 | #define __devexitconst __section(.devexit.rodata) |
91 | 91 | ||
92 | /* Used for HOTPLUG_CPU */ | 92 | /* Used for HOTPLUG_CPU */ |
93 | #define __cpuinit __section(.cpuinit.text) __cold | 93 | #define __cpuinit __section(.cpuinit.text) __cold notrace |
94 | #define __cpuinitdata __section(.cpuinit.data) | 94 | #define __cpuinitdata __section(.cpuinit.data) |
95 | #define __cpuinitconst __section(.cpuinit.rodata) | 95 | #define __cpuinitconst __section(.cpuinit.rodata) |
96 | #define __cpuexit __section(.cpuexit.text) __exitused __cold | 96 | #define __cpuexit __section(.cpuexit.text) __exitused __cold notrace |
97 | #define __cpuexitdata __section(.cpuexit.data) | 97 | #define __cpuexitdata __section(.cpuexit.data) |
98 | #define __cpuexitconst __section(.cpuexit.rodata) | 98 | #define __cpuexitconst __section(.cpuexit.rodata) |
99 | 99 | ||
100 | /* Used for MEMORY_HOTPLUG */ | 100 | /* Used for MEMORY_HOTPLUG */ |
101 | #define __meminit __section(.meminit.text) __cold | 101 | #define __meminit __section(.meminit.text) __cold notrace |
102 | #define __meminitdata __section(.meminit.data) | 102 | #define __meminitdata __section(.meminit.data) |
103 | #define __meminitconst __section(.meminit.rodata) | 103 | #define __meminitconst __section(.meminit.rodata) |
104 | #define __memexit __section(.memexit.text) __exitused __cold | 104 | #define __memexit __section(.memexit.text) __exitused __cold notrace |
105 | #define __memexitdata __section(.memexit.data) | 105 | #define __memexitdata __section(.memexit.data) |
106 | #define __memexitconst __section(.memexit.rodata) | 106 | #define __memexitconst __section(.memexit.rodata) |
107 | 107 | ||
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 7880f18e4b86..83e745f3ead7 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h | |||
@@ -1,20 +1,43 @@ | |||
1 | #ifndef _LINUX_JUMP_LABEL_H | 1 | #ifndef _LINUX_JUMP_LABEL_H |
2 | #define _LINUX_JUMP_LABEL_H | 2 | #define _LINUX_JUMP_LABEL_H |
3 | 3 | ||
4 | #include <linux/types.h> | ||
5 | #include <linux/compiler.h> | ||
6 | |||
4 | #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) | 7 | #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) |
8 | |||
9 | struct jump_label_key { | ||
10 | atomic_t enabled; | ||
11 | struct jump_entry *entries; | ||
12 | #ifdef CONFIG_MODULES | ||
13 | struct jump_label_mod *next; | ||
14 | #endif | ||
15 | }; | ||
16 | |||
5 | # include <asm/jump_label.h> | 17 | # include <asm/jump_label.h> |
6 | # define HAVE_JUMP_LABEL | 18 | # define HAVE_JUMP_LABEL |
7 | #endif | 19 | #endif |
8 | 20 | ||
9 | enum jump_label_type { | 21 | enum jump_label_type { |
22 | JUMP_LABEL_DISABLE = 0, | ||
10 | JUMP_LABEL_ENABLE, | 23 | JUMP_LABEL_ENABLE, |
11 | JUMP_LABEL_DISABLE | ||
12 | }; | 24 | }; |
13 | 25 | ||
14 | struct module; | 26 | struct module; |
15 | 27 | ||
16 | #ifdef HAVE_JUMP_LABEL | 28 | #ifdef HAVE_JUMP_LABEL |
17 | 29 | ||
30 | #ifdef CONFIG_MODULES | ||
31 | #define JUMP_LABEL_INIT {{ 0 }, NULL, NULL} | ||
32 | #else | ||
33 | #define JUMP_LABEL_INIT {{ 0 }, NULL} | ||
34 | #endif | ||
35 | |||
36 | static __always_inline bool static_branch(struct jump_label_key *key) | ||
37 | { | ||
38 | return arch_static_branch(key); | ||
39 | } | ||
40 | |||
18 | extern struct jump_entry __start___jump_table[]; | 41 | extern struct jump_entry __start___jump_table[]; |
19 | extern struct jump_entry __stop___jump_table[]; | 42 | extern struct jump_entry __stop___jump_table[]; |
20 | 43 | ||
@@ -23,37 +46,37 @@ extern void jump_label_unlock(void); | |||
23 | extern void arch_jump_label_transform(struct jump_entry *entry, | 46 | extern void arch_jump_label_transform(struct jump_entry *entry, |
24 | enum jump_label_type type); | 47 | enum jump_label_type type); |
25 | extern void arch_jump_label_text_poke_early(jump_label_t addr); | 48 | extern void arch_jump_label_text_poke_early(jump_label_t addr); |
26 | extern void jump_label_update(unsigned long key, enum jump_label_type type); | ||
27 | extern void jump_label_apply_nops(struct module *mod); | ||
28 | extern int jump_label_text_reserved(void *start, void *end); | 49 | extern int jump_label_text_reserved(void *start, void *end); |
50 | extern void jump_label_inc(struct jump_label_key *key); | ||
51 | extern void jump_label_dec(struct jump_label_key *key); | ||
52 | extern bool jump_label_enabled(struct jump_label_key *key); | ||
53 | extern void jump_label_apply_nops(struct module *mod); | ||
29 | 54 | ||
30 | #define jump_label_enable(key) \ | 55 | #else |
31 | jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE); | ||
32 | 56 | ||
33 | #define jump_label_disable(key) \ | 57 | #include <asm/atomic.h> |
34 | jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE); | ||
35 | 58 | ||
36 | #else | 59 | #define JUMP_LABEL_INIT {ATOMIC_INIT(0)} |
37 | 60 | ||
38 | #define JUMP_LABEL(key, label) \ | 61 | struct jump_label_key { |
39 | do { \ | 62 | atomic_t enabled; |
40 | if (unlikely(*key)) \ | 63 | }; |
41 | goto label; \ | ||
42 | } while (0) | ||
43 | 64 | ||
44 | #define jump_label_enable(cond_var) \ | 65 | static __always_inline bool static_branch(struct jump_label_key *key) |
45 | do { \ | 66 | { |
46 | *(cond_var) = 1; \ | 67 | if (unlikely(atomic_read(&key->enabled))) |
47 | } while (0) | 68 | return true; |
69 | return false; | ||
70 | } | ||
48 | 71 | ||
49 | #define jump_label_disable(cond_var) \ | 72 | static inline void jump_label_inc(struct jump_label_key *key) |
50 | do { \ | 73 | { |
51 | *(cond_var) = 0; \ | 74 | atomic_inc(&key->enabled); |
52 | } while (0) | 75 | } |
53 | 76 | ||
54 | static inline int jump_label_apply_nops(struct module *mod) | 77 | static inline void jump_label_dec(struct jump_label_key *key) |
55 | { | 78 | { |
56 | return 0; | 79 | atomic_dec(&key->enabled); |
57 | } | 80 | } |
58 | 81 | ||
59 | static inline int jump_label_text_reserved(void *start, void *end) | 82 | static inline int jump_label_text_reserved(void *start, void *end) |
@@ -64,16 +87,16 @@ static inline int jump_label_text_reserved(void *start, void *end) | |||
64 | static inline void jump_label_lock(void) {} | 87 | static inline void jump_label_lock(void) {} |
65 | static inline void jump_label_unlock(void) {} | 88 | static inline void jump_label_unlock(void) {} |
66 | 89 | ||
67 | #endif | 90 | static inline bool jump_label_enabled(struct jump_label_key *key) |
91 | { | ||
92 | return !!atomic_read(&key->enabled); | ||
93 | } | ||
68 | 94 | ||
69 | #define COND_STMT(key, stmt) \ | 95 | static inline int jump_label_apply_nops(struct module *mod) |
70 | do { \ | 96 | { |
71 | __label__ jl_enabled; \ | 97 | return 0; |
72 | JUMP_LABEL(key, jl_enabled); \ | 98 | } |
73 | if (0) { \ | 99 | |
74 | jl_enabled: \ | 100 | #endif |
75 | stmt; \ | ||
76 | } \ | ||
77 | } while (0) | ||
78 | 101 | ||
79 | #endif | 102 | #endif |
diff --git a/include/linux/jump_label_ref.h b/include/linux/jump_label_ref.h deleted file mode 100644 index e5d012ad92c6..000000000000 --- a/include/linux/jump_label_ref.h +++ /dev/null | |||
@@ -1,44 +0,0 @@ | |||
1 | #ifndef _LINUX_JUMP_LABEL_REF_H | ||
2 | #define _LINUX_JUMP_LABEL_REF_H | ||
3 | |||
4 | #include <linux/jump_label.h> | ||
5 | #include <asm/atomic.h> | ||
6 | |||
7 | #ifdef HAVE_JUMP_LABEL | ||
8 | |||
9 | static inline void jump_label_inc(atomic_t *key) | ||
10 | { | ||
11 | if (atomic_add_return(1, key) == 1) | ||
12 | jump_label_enable(key); | ||
13 | } | ||
14 | |||
15 | static inline void jump_label_dec(atomic_t *key) | ||
16 | { | ||
17 | if (atomic_dec_and_test(key)) | ||
18 | jump_label_disable(key); | ||
19 | } | ||
20 | |||
21 | #else /* !HAVE_JUMP_LABEL */ | ||
22 | |||
23 | static inline void jump_label_inc(atomic_t *key) | ||
24 | { | ||
25 | atomic_inc(key); | ||
26 | } | ||
27 | |||
28 | static inline void jump_label_dec(atomic_t *key) | ||
29 | { | ||
30 | atomic_dec(key); | ||
31 | } | ||
32 | |||
33 | #undef JUMP_LABEL | ||
34 | #define JUMP_LABEL(key, label) \ | ||
35 | do { \ | ||
36 | if (unlikely(__builtin_choose_expr( \ | ||
37 | __builtin_types_compatible_p(typeof(key), atomic_t *), \ | ||
38 | atomic_read((atomic_t *)(key)), *(key)))) \ | ||
39 | goto label; \ | ||
40 | } while (0) | ||
41 | |||
42 | #endif /* HAVE_JUMP_LABEL */ | ||
43 | |||
44 | #endif /* _LINUX_JUMP_LABEL_REF_H */ | ||
diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 00cec4dc0ae2..f37ba716ef8b 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h | |||
@@ -283,6 +283,7 @@ extern char *get_options(const char *str, int nints, int *ints); | |||
283 | extern unsigned long long memparse(const char *ptr, char **retptr); | 283 | extern unsigned long long memparse(const char *ptr, char **retptr); |
284 | 284 | ||
285 | extern int core_kernel_text(unsigned long addr); | 285 | extern int core_kernel_text(unsigned long addr); |
286 | extern int core_kernel_data(unsigned long addr); | ||
286 | extern int __kernel_text_address(unsigned long addr); | 287 | extern int __kernel_text_address(unsigned long addr); |
287 | extern int kernel_text_address(unsigned long addr); | 288 | extern int kernel_text_address(unsigned long addr); |
288 | extern int func_ptr_is_kernel_text(void *ptr); | 289 | extern int func_ptr_is_kernel_text(void *ptr); |
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index ee9f1e782800..3412684ce5d5 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h | |||
@@ -2,8 +2,8 @@ | |||
2 | * Performance events: | 2 | * Performance events: |
3 | * | 3 | * |
4 | * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de> | 4 | * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de> |
5 | * Copyright (C) 2008-2009, Red Hat, Inc., Ingo Molnar | 5 | * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar |
6 | * Copyright (C) 2008-2009, Red Hat, Inc., Peter Zijlstra | 6 | * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra |
7 | * | 7 | * |
8 | * Data type definitions, declarations, prototypes. | 8 | * Data type definitions, declarations, prototypes. |
9 | * | 9 | * |
@@ -52,6 +52,8 @@ enum perf_hw_id { | |||
52 | PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4, | 52 | PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4, |
53 | PERF_COUNT_HW_BRANCH_MISSES = 5, | 53 | PERF_COUNT_HW_BRANCH_MISSES = 5, |
54 | PERF_COUNT_HW_BUS_CYCLES = 6, | 54 | PERF_COUNT_HW_BUS_CYCLES = 6, |
55 | PERF_COUNT_HW_STALLED_CYCLES_FRONTEND = 7, | ||
56 | PERF_COUNT_HW_STALLED_CYCLES_BACKEND = 8, | ||
55 | 57 | ||
56 | PERF_COUNT_HW_MAX, /* non-ABI */ | 58 | PERF_COUNT_HW_MAX, /* non-ABI */ |
57 | }; | 59 | }; |
@@ -468,9 +470,9 @@ enum perf_callchain_context { | |||
468 | PERF_CONTEXT_MAX = (__u64)-4095, | 470 | PERF_CONTEXT_MAX = (__u64)-4095, |
469 | }; | 471 | }; |
470 | 472 | ||
471 | #define PERF_FLAG_FD_NO_GROUP (1U << 0) | 473 | #define PERF_FLAG_FD_NO_GROUP (1U << 0) |
472 | #define PERF_FLAG_FD_OUTPUT (1U << 1) | 474 | #define PERF_FLAG_FD_OUTPUT (1U << 1) |
473 | #define PERF_FLAG_PID_CGROUP (1U << 2) /* pid=cgroup id, per-cpu mode only */ | 475 | #define PERF_FLAG_PID_CGROUP (1U << 2) /* pid=cgroup id, per-cpu mode only */ |
474 | 476 | ||
475 | #ifdef __KERNEL__ | 477 | #ifdef __KERNEL__ |
476 | /* | 478 | /* |
@@ -484,9 +486,9 @@ enum perf_callchain_context { | |||
484 | #endif | 486 | #endif |
485 | 487 | ||
486 | struct perf_guest_info_callbacks { | 488 | struct perf_guest_info_callbacks { |
487 | int (*is_in_guest) (void); | 489 | int (*is_in_guest)(void); |
488 | int (*is_user_mode) (void); | 490 | int (*is_user_mode)(void); |
489 | unsigned long (*get_guest_ip) (void); | 491 | unsigned long (*get_guest_ip)(void); |
490 | }; | 492 | }; |
491 | 493 | ||
492 | #ifdef CONFIG_HAVE_HW_BREAKPOINT | 494 | #ifdef CONFIG_HAVE_HW_BREAKPOINT |
@@ -505,7 +507,7 @@ struct perf_guest_info_callbacks { | |||
505 | #include <linux/ftrace.h> | 507 | #include <linux/ftrace.h> |
506 | #include <linux/cpu.h> | 508 | #include <linux/cpu.h> |
507 | #include <linux/irq_work.h> | 509 | #include <linux/irq_work.h> |
508 | #include <linux/jump_label_ref.h> | 510 | #include <linux/jump_label.h> |
509 | #include <asm/atomic.h> | 511 | #include <asm/atomic.h> |
510 | #include <asm/local.h> | 512 | #include <asm/local.h> |
511 | 513 | ||
@@ -652,19 +654,19 @@ struct pmu { | |||
652 | * Start the transaction, after this ->add() doesn't need to | 654 | * Start the transaction, after this ->add() doesn't need to |
653 | * do schedulability tests. | 655 | * do schedulability tests. |
654 | */ | 656 | */ |
655 | void (*start_txn) (struct pmu *pmu); /* optional */ | 657 | void (*start_txn) (struct pmu *pmu); /* optional */ |
656 | /* | 658 | /* |
657 | * If ->start_txn() disabled the ->add() schedulability test | 659 | * If ->start_txn() disabled the ->add() schedulability test |
658 | * then ->commit_txn() is required to perform one. On success | 660 | * then ->commit_txn() is required to perform one. On success |
659 | * the transaction is closed. On error the transaction is kept | 661 | * the transaction is closed. On error the transaction is kept |
660 | * open until ->cancel_txn() is called. | 662 | * open until ->cancel_txn() is called. |
661 | */ | 663 | */ |
662 | int (*commit_txn) (struct pmu *pmu); /* optional */ | 664 | int (*commit_txn) (struct pmu *pmu); /* optional */ |
663 | /* | 665 | /* |
664 | * Will cancel the transaction, assumes ->del() is called | 666 | * Will cancel the transaction, assumes ->del() is called |
665 | * for each successful ->add() during the transaction. | 667 | * for each successful ->add() during the transaction. |
666 | */ | 668 | */ |
667 | void (*cancel_txn) (struct pmu *pmu); /* optional */ | 669 | void (*cancel_txn) (struct pmu *pmu); /* optional */ |
668 | }; | 670 | }; |
669 | 671 | ||
670 | /** | 672 | /** |
@@ -712,15 +714,15 @@ typedef void (*perf_overflow_handler_t)(struct perf_event *, int, | |||
712 | struct pt_regs *regs); | 714 | struct pt_regs *regs); |
713 | 715 | ||
714 | enum perf_group_flag { | 716 | enum perf_group_flag { |
715 | PERF_GROUP_SOFTWARE = 0x1, | 717 | PERF_GROUP_SOFTWARE = 0x1, |
716 | }; | 718 | }; |
717 | 719 | ||
718 | #define SWEVENT_HLIST_BITS 8 | 720 | #define SWEVENT_HLIST_BITS 8 |
719 | #define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS) | 721 | #define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS) |
720 | 722 | ||
721 | struct swevent_hlist { | 723 | struct swevent_hlist { |
722 | struct hlist_head heads[SWEVENT_HLIST_SIZE]; | 724 | struct hlist_head heads[SWEVENT_HLIST_SIZE]; |
723 | struct rcu_head rcu_head; | 725 | struct rcu_head rcu_head; |
724 | }; | 726 | }; |
725 | 727 | ||
726 | #define PERF_ATTACH_CONTEXT 0x01 | 728 | #define PERF_ATTACH_CONTEXT 0x01 |
@@ -733,13 +735,13 @@ struct swevent_hlist { | |||
733 | * This is a per-cpu dynamically allocated data structure. | 735 | * This is a per-cpu dynamically allocated data structure. |
734 | */ | 736 | */ |
735 | struct perf_cgroup_info { | 737 | struct perf_cgroup_info { |
736 | u64 time; | 738 | u64 time; |
737 | u64 timestamp; | 739 | u64 timestamp; |
738 | }; | 740 | }; |
739 | 741 | ||
740 | struct perf_cgroup { | 742 | struct perf_cgroup { |
741 | struct cgroup_subsys_state css; | 743 | struct cgroup_subsys_state css; |
742 | struct perf_cgroup_info *info; /* timing info, one per cpu */ | 744 | struct perf_cgroup_info *info; /* timing info, one per cpu */ |
743 | }; | 745 | }; |
744 | #endif | 746 | #endif |
745 | 747 | ||
@@ -923,7 +925,7 @@ struct perf_event_context { | |||
923 | 925 | ||
924 | /* | 926 | /* |
925 | * Number of contexts where an event can trigger: | 927 | * Number of contexts where an event can trigger: |
926 | * task, softirq, hardirq, nmi. | 928 | * task, softirq, hardirq, nmi. |
927 | */ | 929 | */ |
928 | #define PERF_NR_CONTEXTS 4 | 930 | #define PERF_NR_CONTEXTS 4 |
929 | 931 | ||
@@ -1001,8 +1003,7 @@ struct perf_sample_data { | |||
1001 | struct perf_raw_record *raw; | 1003 | struct perf_raw_record *raw; |
1002 | }; | 1004 | }; |
1003 | 1005 | ||
1004 | static inline | 1006 | static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr) |
1005 | void perf_sample_data_init(struct perf_sample_data *data, u64 addr) | ||
1006 | { | 1007 | { |
1007 | data->addr = addr; | 1008 | data->addr = addr; |
1008 | data->raw = NULL; | 1009 | data->raw = NULL; |
@@ -1034,13 +1035,12 @@ static inline int is_software_event(struct perf_event *event) | |||
1034 | return event->pmu->task_ctx_nr == perf_sw_context; | 1035 | return event->pmu->task_ctx_nr == perf_sw_context; |
1035 | } | 1036 | } |
1036 | 1037 | ||
1037 | extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; | 1038 | extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; |
1038 | 1039 | ||
1039 | extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64); | 1040 | extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64); |
1040 | 1041 | ||
1041 | #ifndef perf_arch_fetch_caller_regs | 1042 | #ifndef perf_arch_fetch_caller_regs |
1042 | static inline void | 1043 | static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { } |
1043 | perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { } | ||
1044 | #endif | 1044 | #endif |
1045 | 1045 | ||
1046 | /* | 1046 | /* |
@@ -1063,26 +1063,24 @@ perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr) | |||
1063 | { | 1063 | { |
1064 | struct pt_regs hot_regs; | 1064 | struct pt_regs hot_regs; |
1065 | 1065 | ||
1066 | JUMP_LABEL(&perf_swevent_enabled[event_id], have_event); | 1066 | if (static_branch(&perf_swevent_enabled[event_id])) { |
1067 | return; | 1067 | if (!regs) { |
1068 | 1068 | perf_fetch_caller_regs(&hot_regs); | |
1069 | have_event: | 1069 | regs = &hot_regs; |
1070 | if (!regs) { | 1070 | } |
1071 | perf_fetch_caller_regs(&hot_regs); | 1071 | __perf_sw_event(event_id, nr, nmi, regs, addr); |
1072 | regs = &hot_regs; | ||
1073 | } | 1072 | } |
1074 | __perf_sw_event(event_id, nr, nmi, regs, addr); | ||
1075 | } | 1073 | } |
1076 | 1074 | ||
1077 | extern atomic_t perf_sched_events; | 1075 | extern struct jump_label_key perf_sched_events; |
1078 | 1076 | ||
1079 | static inline void perf_event_task_sched_in(struct task_struct *task) | 1077 | static inline void perf_event_task_sched_in(struct task_struct *task) |
1080 | { | 1078 | { |
1081 | COND_STMT(&perf_sched_events, __perf_event_task_sched_in(task)); | 1079 | if (static_branch(&perf_sched_events)) |
1080 | __perf_event_task_sched_in(task); | ||
1082 | } | 1081 | } |
1083 | 1082 | ||
1084 | static inline | 1083 | static inline void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next) |
1085 | void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next) | ||
1086 | { | 1084 | { |
1087 | perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0); | 1085 | perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0); |
1088 | 1086 | ||
@@ -1100,14 +1098,10 @@ extern void perf_event_fork(struct task_struct *tsk); | |||
1100 | /* Callchains */ | 1098 | /* Callchains */ |
1101 | DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); | 1099 | DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); |
1102 | 1100 | ||
1103 | extern void perf_callchain_user(struct perf_callchain_entry *entry, | 1101 | extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs); |
1104 | struct pt_regs *regs); | 1102 | extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs); |
1105 | extern void perf_callchain_kernel(struct perf_callchain_entry *entry, | ||
1106 | struct pt_regs *regs); | ||
1107 | |||
1108 | 1103 | ||
1109 | static inline void | 1104 | static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip) |
1110 | perf_callchain_store(struct perf_callchain_entry *entry, u64 ip) | ||
1111 | { | 1105 | { |
1112 | if (entry->nr < PERF_MAX_STACK_DEPTH) | 1106 | if (entry->nr < PERF_MAX_STACK_DEPTH) |
1113 | entry->ip[entry->nr++] = ip; | 1107 | entry->ip[entry->nr++] = ip; |
@@ -1143,9 +1137,9 @@ extern void perf_tp_event(u64 addr, u64 count, void *record, | |||
1143 | extern void perf_bp_event(struct perf_event *event, void *data); | 1137 | extern void perf_bp_event(struct perf_event *event, void *data); |
1144 | 1138 | ||
1145 | #ifndef perf_misc_flags | 1139 | #ifndef perf_misc_flags |
1146 | #define perf_misc_flags(regs) (user_mode(regs) ? PERF_RECORD_MISC_USER : \ | 1140 | # define perf_misc_flags(regs) \ |
1147 | PERF_RECORD_MISC_KERNEL) | 1141 | (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL) |
1148 | #define perf_instruction_pointer(regs) instruction_pointer(regs) | 1142 | # define perf_instruction_pointer(regs) instruction_pointer(regs) |
1149 | #endif | 1143 | #endif |
1150 | 1144 | ||
1151 | extern int perf_output_begin(struct perf_output_handle *handle, | 1145 | extern int perf_output_begin(struct perf_output_handle *handle, |
@@ -1180,9 +1174,9 @@ static inline void | |||
1180 | perf_bp_event(struct perf_event *event, void *data) { } | 1174 | perf_bp_event(struct perf_event *event, void *data) { } |
1181 | 1175 | ||
1182 | static inline int perf_register_guest_info_callbacks | 1176 | static inline int perf_register_guest_info_callbacks |
1183 | (struct perf_guest_info_callbacks *callbacks) { return 0; } | 1177 | (struct perf_guest_info_callbacks *callbacks) { return 0; } |
1184 | static inline int perf_unregister_guest_info_callbacks | 1178 | static inline int perf_unregister_guest_info_callbacks |
1185 | (struct perf_guest_info_callbacks *callbacks) { return 0; } | 1179 | (struct perf_guest_info_callbacks *callbacks) { return 0; } |
1186 | 1180 | ||
1187 | static inline void perf_event_mmap(struct vm_area_struct *vma) { } | 1181 | static inline void perf_event_mmap(struct vm_area_struct *vma) { } |
1188 | static inline void perf_event_comm(struct task_struct *tsk) { } | 1182 | static inline void perf_event_comm(struct task_struct *tsk) { } |
@@ -1195,23 +1189,22 @@ static inline void perf_event_disable(struct perf_event *event) { } | |||
1195 | static inline void perf_event_task_tick(void) { } | 1189 | static inline void perf_event_task_tick(void) { } |
1196 | #endif | 1190 | #endif |
1197 | 1191 | ||
1198 | #define perf_output_put(handle, x) \ | 1192 | #define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x)) |
1199 | perf_output_copy((handle), &(x), sizeof(x)) | ||
1200 | 1193 | ||
1201 | /* | 1194 | /* |
1202 | * This has to have a higher priority than migration_notifier in sched.c. | 1195 | * This has to have a higher priority than migration_notifier in sched.c. |
1203 | */ | 1196 | */ |
1204 | #define perf_cpu_notifier(fn) \ | 1197 | #define perf_cpu_notifier(fn) \ |
1205 | do { \ | 1198 | do { \ |
1206 | static struct notifier_block fn##_nb __cpuinitdata = \ | 1199 | static struct notifier_block fn##_nb __cpuinitdata = \ |
1207 | { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ | 1200 | { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ |
1208 | fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ | 1201 | fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ |
1209 | (void *)(unsigned long)smp_processor_id()); \ | 1202 | (void *)(unsigned long)smp_processor_id()); \ |
1210 | fn(&fn##_nb, (unsigned long)CPU_STARTING, \ | 1203 | fn(&fn##_nb, (unsigned long)CPU_STARTING, \ |
1211 | (void *)(unsigned long)smp_processor_id()); \ | 1204 | (void *)(unsigned long)smp_processor_id()); \ |
1212 | fn(&fn##_nb, (unsigned long)CPU_ONLINE, \ | 1205 | fn(&fn##_nb, (unsigned long)CPU_ONLINE, \ |
1213 | (void *)(unsigned long)smp_processor_id()); \ | 1206 | (void *)(unsigned long)smp_processor_id()); \ |
1214 | register_cpu_notifier(&fn##_nb); \ | 1207 | register_cpu_notifier(&fn##_nb); \ |
1215 | } while (0) | 1208 | } while (0) |
1216 | 1209 | ||
1217 | #endif /* __KERNEL__ */ | 1210 | #endif /* __KERNEL__ */ |
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 97c84a58efb8..d530a4460a0b 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h | |||
@@ -29,7 +29,7 @@ struct tracepoint_func { | |||
29 | 29 | ||
30 | struct tracepoint { | 30 | struct tracepoint { |
31 | const char *name; /* Tracepoint name */ | 31 | const char *name; /* Tracepoint name */ |
32 | int state; /* State. */ | 32 | struct jump_label_key key; |
33 | void (*regfunc)(void); | 33 | void (*regfunc)(void); |
34 | void (*unregfunc)(void); | 34 | void (*unregfunc)(void); |
35 | struct tracepoint_func __rcu *funcs; | 35 | struct tracepoint_func __rcu *funcs; |
@@ -146,9 +146,7 @@ void tracepoint_update_probe_range(struct tracepoint * const *begin, | |||
146 | extern struct tracepoint __tracepoint_##name; \ | 146 | extern struct tracepoint __tracepoint_##name; \ |
147 | static inline void trace_##name(proto) \ | 147 | static inline void trace_##name(proto) \ |
148 | { \ | 148 | { \ |
149 | JUMP_LABEL(&__tracepoint_##name.state, do_trace); \ | 149 | if (static_branch(&__tracepoint_##name.key)) \ |
150 | return; \ | ||
151 | do_trace: \ | ||
152 | __DO_TRACE(&__tracepoint_##name, \ | 150 | __DO_TRACE(&__tracepoint_##name, \ |
153 | TP_PROTO(data_proto), \ | 151 | TP_PROTO(data_proto), \ |
154 | TP_ARGS(data_args), \ | 152 | TP_ARGS(data_args), \ |
@@ -176,14 +174,14 @@ do_trace: \ | |||
176 | * structures, so we create an array of pointers that will be used for iteration | 174 | * structures, so we create an array of pointers that will be used for iteration |
177 | * on the tracepoints. | 175 | * on the tracepoints. |
178 | */ | 176 | */ |
179 | #define DEFINE_TRACE_FN(name, reg, unreg) \ | 177 | #define DEFINE_TRACE_FN(name, reg, unreg) \ |
180 | static const char __tpstrtab_##name[] \ | 178 | static const char __tpstrtab_##name[] \ |
181 | __attribute__((section("__tracepoints_strings"))) = #name; \ | 179 | __attribute__((section("__tracepoints_strings"))) = #name; \ |
182 | struct tracepoint __tracepoint_##name \ | 180 | struct tracepoint __tracepoint_##name \ |
183 | __attribute__((section("__tracepoints"))) = \ | 181 | __attribute__((section("__tracepoints"))) = \ |
184 | { __tpstrtab_##name, 0, reg, unreg, NULL }; \ | 182 | { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\ |
185 | static struct tracepoint * const __tracepoint_ptr_##name __used \ | 183 | static struct tracepoint * const __tracepoint_ptr_##name __used \ |
186 | __attribute__((section("__tracepoints_ptrs"))) = \ | 184 | __attribute__((section("__tracepoints_ptrs"))) = \ |
187 | &__tracepoint_##name; | 185 | &__tracepoint_##name; |
188 | 186 | ||
189 | #define DEFINE_TRACE(name) \ | 187 | #define DEFINE_TRACE(name) \ |
diff --git a/kernel/Makefile b/kernel/Makefile index 85cbfb31e73e..e9cf19155b46 100644 --- a/kernel/Makefile +++ b/kernel/Makefile | |||
@@ -21,7 +21,6 @@ CFLAGS_REMOVE_mutex-debug.o = -pg | |||
21 | CFLAGS_REMOVE_rtmutex-debug.o = -pg | 21 | CFLAGS_REMOVE_rtmutex-debug.o = -pg |
22 | CFLAGS_REMOVE_cgroup-debug.o = -pg | 22 | CFLAGS_REMOVE_cgroup-debug.o = -pg |
23 | CFLAGS_REMOVE_sched_clock.o = -pg | 23 | CFLAGS_REMOVE_sched_clock.o = -pg |
24 | CFLAGS_REMOVE_perf_event.o = -pg | ||
25 | CFLAGS_REMOVE_irq_work.o = -pg | 24 | CFLAGS_REMOVE_irq_work.o = -pg |
26 | endif | 25 | endif |
27 | 26 | ||
@@ -103,8 +102,9 @@ obj-$(CONFIG_RING_BUFFER) += trace/ | |||
103 | obj-$(CONFIG_TRACEPOINTS) += trace/ | 102 | obj-$(CONFIG_TRACEPOINTS) += trace/ |
104 | obj-$(CONFIG_SMP) += sched_cpupri.o | 103 | obj-$(CONFIG_SMP) += sched_cpupri.o |
105 | obj-$(CONFIG_IRQ_WORK) += irq_work.o | 104 | obj-$(CONFIG_IRQ_WORK) += irq_work.o |
106 | obj-$(CONFIG_PERF_EVENTS) += perf_event.o | 105 | |
107 | obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o | 106 | obj-$(CONFIG_PERF_EVENTS) += events/ |
107 | |||
108 | obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o | 108 | obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o |
109 | obj-$(CONFIG_PADATA) += padata.o | 109 | obj-$(CONFIG_PADATA) += padata.o |
110 | obj-$(CONFIG_CRASH_DUMP) += crash_dump.o | 110 | obj-$(CONFIG_CRASH_DUMP) += crash_dump.o |
diff --git a/kernel/events/Makefile b/kernel/events/Makefile new file mode 100644 index 000000000000..1ce23d3d8394 --- /dev/null +++ b/kernel/events/Makefile | |||
@@ -0,0 +1,6 @@ | |||
1 | ifdef CONFIG_FUNCTION_TRACER | ||
2 | CFLAGS_REMOVE_core.o = -pg | ||
3 | endif | ||
4 | |||
5 | obj-y := core.o | ||
6 | obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o | ||
diff --git a/kernel/perf_event.c b/kernel/events/core.c index 8e81a9860a0d..0fc34a370ba4 100644 --- a/kernel/perf_event.c +++ b/kernel/events/core.c | |||
@@ -2,8 +2,8 @@ | |||
2 | * Performance events core code: | 2 | * Performance events core code: |
3 | * | 3 | * |
4 | * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> | 4 | * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> |
5 | * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar | 5 | * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar |
6 | * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> | 6 | * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> |
7 | * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> | 7 | * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> |
8 | * | 8 | * |
9 | * For licensing details see kernel-base/COPYING | 9 | * For licensing details see kernel-base/COPYING |
@@ -39,10 +39,10 @@ | |||
39 | #include <asm/irq_regs.h> | 39 | #include <asm/irq_regs.h> |
40 | 40 | ||
41 | struct remote_function_call { | 41 | struct remote_function_call { |
42 | struct task_struct *p; | 42 | struct task_struct *p; |
43 | int (*func)(void *info); | 43 | int (*func)(void *info); |
44 | void *info; | 44 | void *info; |
45 | int ret; | 45 | int ret; |
46 | }; | 46 | }; |
47 | 47 | ||
48 | static void remote_function(void *data) | 48 | static void remote_function(void *data) |
@@ -76,10 +76,10 @@ static int | |||
76 | task_function_call(struct task_struct *p, int (*func) (void *info), void *info) | 76 | task_function_call(struct task_struct *p, int (*func) (void *info), void *info) |
77 | { | 77 | { |
78 | struct remote_function_call data = { | 78 | struct remote_function_call data = { |
79 | .p = p, | 79 | .p = p, |
80 | .func = func, | 80 | .func = func, |
81 | .info = info, | 81 | .info = info, |
82 | .ret = -ESRCH, /* No such (running) process */ | 82 | .ret = -ESRCH, /* No such (running) process */ |
83 | }; | 83 | }; |
84 | 84 | ||
85 | if (task_curr(p)) | 85 | if (task_curr(p)) |
@@ -100,10 +100,10 @@ task_function_call(struct task_struct *p, int (*func) (void *info), void *info) | |||
100 | static int cpu_function_call(int cpu, int (*func) (void *info), void *info) | 100 | static int cpu_function_call(int cpu, int (*func) (void *info), void *info) |
101 | { | 101 | { |
102 | struct remote_function_call data = { | 102 | struct remote_function_call data = { |
103 | .p = NULL, | 103 | .p = NULL, |
104 | .func = func, | 104 | .func = func, |
105 | .info = info, | 105 | .info = info, |
106 | .ret = -ENXIO, /* No such CPU */ | 106 | .ret = -ENXIO, /* No such CPU */ |
107 | }; | 107 | }; |
108 | 108 | ||
109 | smp_call_function_single(cpu, remote_function, &data, 1); | 109 | smp_call_function_single(cpu, remote_function, &data, 1); |
@@ -125,7 +125,7 @@ enum event_type_t { | |||
125 | * perf_sched_events : >0 events exist | 125 | * perf_sched_events : >0 events exist |
126 | * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu | 126 | * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu |
127 | */ | 127 | */ |
128 | atomic_t perf_sched_events __read_mostly; | 128 | struct jump_label_key perf_sched_events __read_mostly; |
129 | static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); | 129 | static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); |
130 | 130 | ||
131 | static atomic_t nr_mmap_events __read_mostly; | 131 | static atomic_t nr_mmap_events __read_mostly; |
@@ -5429,7 +5429,7 @@ fail: | |||
5429 | return err; | 5429 | return err; |
5430 | } | 5430 | } |
5431 | 5431 | ||
5432 | atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; | 5432 | struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; |
5433 | 5433 | ||
5434 | static void sw_perf_event_destroy(struct perf_event *event) | 5434 | static void sw_perf_event_destroy(struct perf_event *event) |
5435 | { | 5435 | { |
@@ -7445,11 +7445,11 @@ static void perf_cgroup_exit(struct cgroup_subsys *ss, struct cgroup *cgrp, | |||
7445 | } | 7445 | } |
7446 | 7446 | ||
7447 | struct cgroup_subsys perf_subsys = { | 7447 | struct cgroup_subsys perf_subsys = { |
7448 | .name = "perf_event", | 7448 | .name = "perf_event", |
7449 | .subsys_id = perf_subsys_id, | 7449 | .subsys_id = perf_subsys_id, |
7450 | .create = perf_cgroup_create, | 7450 | .create = perf_cgroup_create, |
7451 | .destroy = perf_cgroup_destroy, | 7451 | .destroy = perf_cgroup_destroy, |
7452 | .exit = perf_cgroup_exit, | 7452 | .exit = perf_cgroup_exit, |
7453 | .attach = perf_cgroup_attach, | 7453 | .attach = perf_cgroup_attach, |
7454 | }; | 7454 | }; |
7455 | #endif /* CONFIG_CGROUP_PERF */ | 7455 | #endif /* CONFIG_CGROUP_PERF */ |
diff --git a/kernel/hw_breakpoint.c b/kernel/events/hw_breakpoint.c index 086adf25a55e..086adf25a55e 100644 --- a/kernel/hw_breakpoint.c +++ b/kernel/events/hw_breakpoint.c | |||
diff --git a/kernel/extable.c b/kernel/extable.c index 7f8f263f8524..c2d625fcda77 100644 --- a/kernel/extable.c +++ b/kernel/extable.c | |||
@@ -72,6 +72,14 @@ int core_kernel_text(unsigned long addr) | |||
72 | return 0; | 72 | return 0; |
73 | } | 73 | } |
74 | 74 | ||
75 | int core_kernel_data(unsigned long addr) | ||
76 | { | ||
77 | if (addr >= (unsigned long)_sdata && | ||
78 | addr < (unsigned long)_edata) | ||
79 | return 1; | ||
80 | return 0; | ||
81 | } | ||
82 | |||
75 | int __kernel_text_address(unsigned long addr) | 83 | int __kernel_text_address(unsigned long addr) |
76 | { | 84 | { |
77 | if (core_kernel_text(addr)) | 85 | if (core_kernel_text(addr)) |
diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 3b79bd938330..74d1c099fbd1 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c | |||
@@ -2,43 +2,23 @@ | |||
2 | * jump label support | 2 | * jump label support |
3 | * | 3 | * |
4 | * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> | 4 | * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> |
5 | * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com> | ||
5 | * | 6 | * |
6 | */ | 7 | */ |
7 | #include <linux/jump_label.h> | ||
8 | #include <linux/memory.h> | 8 | #include <linux/memory.h> |
9 | #include <linux/uaccess.h> | 9 | #include <linux/uaccess.h> |
10 | #include <linux/module.h> | 10 | #include <linux/module.h> |
11 | #include <linux/list.h> | 11 | #include <linux/list.h> |
12 | #include <linux/jhash.h> | ||
13 | #include <linux/slab.h> | 12 | #include <linux/slab.h> |
14 | #include <linux/sort.h> | 13 | #include <linux/sort.h> |
15 | #include <linux/err.h> | 14 | #include <linux/err.h> |
15 | #include <linux/jump_label.h> | ||
16 | 16 | ||
17 | #ifdef HAVE_JUMP_LABEL | 17 | #ifdef HAVE_JUMP_LABEL |
18 | 18 | ||
19 | #define JUMP_LABEL_HASH_BITS 6 | ||
20 | #define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS) | ||
21 | static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE]; | ||
22 | |||
23 | /* mutex to protect coming/going of the the jump_label table */ | 19 | /* mutex to protect coming/going of the the jump_label table */ |
24 | static DEFINE_MUTEX(jump_label_mutex); | 20 | static DEFINE_MUTEX(jump_label_mutex); |
25 | 21 | ||
26 | struct jump_label_entry { | ||
27 | struct hlist_node hlist; | ||
28 | struct jump_entry *table; | ||
29 | int nr_entries; | ||
30 | /* hang modules off here */ | ||
31 | struct hlist_head modules; | ||
32 | unsigned long key; | ||
33 | }; | ||
34 | |||
35 | struct jump_label_module_entry { | ||
36 | struct hlist_node hlist; | ||
37 | struct jump_entry *table; | ||
38 | int nr_entries; | ||
39 | struct module *mod; | ||
40 | }; | ||
41 | |||
42 | void jump_label_lock(void) | 22 | void jump_label_lock(void) |
43 | { | 23 | { |
44 | mutex_lock(&jump_label_mutex); | 24 | mutex_lock(&jump_label_mutex); |
@@ -49,6 +29,11 @@ void jump_label_unlock(void) | |||
49 | mutex_unlock(&jump_label_mutex); | 29 | mutex_unlock(&jump_label_mutex); |
50 | } | 30 | } |
51 | 31 | ||
32 | bool jump_label_enabled(struct jump_label_key *key) | ||
33 | { | ||
34 | return !!atomic_read(&key->enabled); | ||
35 | } | ||
36 | |||
52 | static int jump_label_cmp(const void *a, const void *b) | 37 | static int jump_label_cmp(const void *a, const void *b) |
53 | { | 38 | { |
54 | const struct jump_entry *jea = a; | 39 | const struct jump_entry *jea = a; |
@@ -64,7 +49,7 @@ static int jump_label_cmp(const void *a, const void *b) | |||
64 | } | 49 | } |
65 | 50 | ||
66 | static void | 51 | static void |
67 | sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop) | 52 | jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) |
68 | { | 53 | { |
69 | unsigned long size; | 54 | unsigned long size; |
70 | 55 | ||
@@ -73,118 +58,25 @@ sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop) | |||
73 | sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); | 58 | sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); |
74 | } | 59 | } |
75 | 60 | ||
76 | static struct jump_label_entry *get_jump_label_entry(jump_label_t key) | 61 | static void jump_label_update(struct jump_label_key *key, int enable); |
77 | { | ||
78 | struct hlist_head *head; | ||
79 | struct hlist_node *node; | ||
80 | struct jump_label_entry *e; | ||
81 | u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0); | ||
82 | |||
83 | head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; | ||
84 | hlist_for_each_entry(e, node, head, hlist) { | ||
85 | if (key == e->key) | ||
86 | return e; | ||
87 | } | ||
88 | return NULL; | ||
89 | } | ||
90 | 62 | ||
91 | static struct jump_label_entry * | 63 | void jump_label_inc(struct jump_label_key *key) |
92 | add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table) | ||
93 | { | 64 | { |
94 | struct hlist_head *head; | 65 | if (atomic_inc_not_zero(&key->enabled)) |
95 | struct jump_label_entry *e; | 66 | return; |
96 | u32 hash; | ||
97 | |||
98 | e = get_jump_label_entry(key); | ||
99 | if (e) | ||
100 | return ERR_PTR(-EEXIST); | ||
101 | |||
102 | e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL); | ||
103 | if (!e) | ||
104 | return ERR_PTR(-ENOMEM); | ||
105 | |||
106 | hash = jhash((void *)&key, sizeof(jump_label_t), 0); | ||
107 | head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; | ||
108 | e->key = key; | ||
109 | e->table = table; | ||
110 | e->nr_entries = nr_entries; | ||
111 | INIT_HLIST_HEAD(&(e->modules)); | ||
112 | hlist_add_head(&e->hlist, head); | ||
113 | return e; | ||
114 | } | ||
115 | 67 | ||
116 | static int | 68 | jump_label_lock(); |
117 | build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop) | 69 | if (atomic_add_return(1, &key->enabled) == 1) |
118 | { | 70 | jump_label_update(key, JUMP_LABEL_ENABLE); |
119 | struct jump_entry *iter, *iter_begin; | 71 | jump_label_unlock(); |
120 | struct jump_label_entry *entry; | ||
121 | int count; | ||
122 | |||
123 | sort_jump_label_entries(start, stop); | ||
124 | iter = start; | ||
125 | while (iter < stop) { | ||
126 | entry = get_jump_label_entry(iter->key); | ||
127 | if (!entry) { | ||
128 | iter_begin = iter; | ||
129 | count = 0; | ||
130 | while ((iter < stop) && | ||
131 | (iter->key == iter_begin->key)) { | ||
132 | iter++; | ||
133 | count++; | ||
134 | } | ||
135 | entry = add_jump_label_entry(iter_begin->key, | ||
136 | count, iter_begin); | ||
137 | if (IS_ERR(entry)) | ||
138 | return PTR_ERR(entry); | ||
139 | } else { | ||
140 | WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n"); | ||
141 | return -1; | ||
142 | } | ||
143 | } | ||
144 | return 0; | ||
145 | } | 72 | } |
146 | 73 | ||
147 | /*** | 74 | void jump_label_dec(struct jump_label_key *key) |
148 | * jump_label_update - update jump label text | ||
149 | * @key - key value associated with a a jump label | ||
150 | * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE | ||
151 | * | ||
152 | * Will enable/disable the jump for jump label @key, depending on the | ||
153 | * value of @type. | ||
154 | * | ||
155 | */ | ||
156 | |||
157 | void jump_label_update(unsigned long key, enum jump_label_type type) | ||
158 | { | 75 | { |
159 | struct jump_entry *iter; | 76 | if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) |
160 | struct jump_label_entry *entry; | 77 | return; |
161 | struct hlist_node *module_node; | ||
162 | struct jump_label_module_entry *e_module; | ||
163 | int count; | ||
164 | 78 | ||
165 | jump_label_lock(); | 79 | jump_label_update(key, JUMP_LABEL_DISABLE); |
166 | entry = get_jump_label_entry((jump_label_t)key); | ||
167 | if (entry) { | ||
168 | count = entry->nr_entries; | ||
169 | iter = entry->table; | ||
170 | while (count--) { | ||
171 | if (kernel_text_address(iter->code)) | ||
172 | arch_jump_label_transform(iter, type); | ||
173 | iter++; | ||
174 | } | ||
175 | /* eanble/disable jump labels in modules */ | ||
176 | hlist_for_each_entry(e_module, module_node, &(entry->modules), | ||
177 | hlist) { | ||
178 | count = e_module->nr_entries; | ||
179 | iter = e_module->table; | ||
180 | while (count--) { | ||
181 | if (iter->key && | ||
182 | kernel_text_address(iter->code)) | ||
183 | arch_jump_label_transform(iter, type); | ||
184 | iter++; | ||
185 | } | ||
186 | } | ||
187 | } | ||
188 | jump_label_unlock(); | 80 | jump_label_unlock(); |
189 | } | 81 | } |
190 | 82 | ||
@@ -197,77 +89,33 @@ static int addr_conflict(struct jump_entry *entry, void *start, void *end) | |||
197 | return 0; | 89 | return 0; |
198 | } | 90 | } |
199 | 91 | ||
200 | #ifdef CONFIG_MODULES | 92 | static int __jump_label_text_reserved(struct jump_entry *iter_start, |
201 | 93 | struct jump_entry *iter_stop, void *start, void *end) | |
202 | static int module_conflict(void *start, void *end) | ||
203 | { | 94 | { |
204 | struct hlist_head *head; | ||
205 | struct hlist_node *node, *node_next, *module_node, *module_node_next; | ||
206 | struct jump_label_entry *e; | ||
207 | struct jump_label_module_entry *e_module; | ||
208 | struct jump_entry *iter; | 95 | struct jump_entry *iter; |
209 | int i, count; | ||
210 | int conflict = 0; | ||
211 | |||
212 | for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { | ||
213 | head = &jump_label_table[i]; | ||
214 | hlist_for_each_entry_safe(e, node, node_next, head, hlist) { | ||
215 | hlist_for_each_entry_safe(e_module, module_node, | ||
216 | module_node_next, | ||
217 | &(e->modules), hlist) { | ||
218 | count = e_module->nr_entries; | ||
219 | iter = e_module->table; | ||
220 | while (count--) { | ||
221 | if (addr_conflict(iter, start, end)) { | ||
222 | conflict = 1; | ||
223 | goto out; | ||
224 | } | ||
225 | iter++; | ||
226 | } | ||
227 | } | ||
228 | } | ||
229 | } | ||
230 | out: | ||
231 | return conflict; | ||
232 | } | ||
233 | |||
234 | #endif | ||
235 | |||
236 | /*** | ||
237 | * jump_label_text_reserved - check if addr range is reserved | ||
238 | * @start: start text addr | ||
239 | * @end: end text addr | ||
240 | * | ||
241 | * checks if the text addr located between @start and @end | ||
242 | * overlaps with any of the jump label patch addresses. Code | ||
243 | * that wants to modify kernel text should first verify that | ||
244 | * it does not overlap with any of the jump label addresses. | ||
245 | * Caller must hold jump_label_mutex. | ||
246 | * | ||
247 | * returns 1 if there is an overlap, 0 otherwise | ||
248 | */ | ||
249 | int jump_label_text_reserved(void *start, void *end) | ||
250 | { | ||
251 | struct jump_entry *iter; | ||
252 | struct jump_entry *iter_start = __start___jump_table; | ||
253 | struct jump_entry *iter_stop = __start___jump_table; | ||
254 | int conflict = 0; | ||
255 | 96 | ||
256 | iter = iter_start; | 97 | iter = iter_start; |
257 | while (iter < iter_stop) { | 98 | while (iter < iter_stop) { |
258 | if (addr_conflict(iter, start, end)) { | 99 | if (addr_conflict(iter, start, end)) |
259 | conflict = 1; | 100 | return 1; |
260 | goto out; | ||
261 | } | ||
262 | iter++; | 101 | iter++; |
263 | } | 102 | } |
264 | 103 | ||
265 | /* now check modules */ | 104 | return 0; |
266 | #ifdef CONFIG_MODULES | 105 | } |
267 | conflict = module_conflict(start, end); | 106 | |
268 | #endif | 107 | static void __jump_label_update(struct jump_label_key *key, |
269 | out: | 108 | struct jump_entry *entry, int enable) |
270 | return conflict; | 109 | { |
110 | for (; entry->key == (jump_label_t)(unsigned long)key; entry++) { | ||
111 | /* | ||
112 | * entry->code set to 0 invalidates module init text sections | ||
113 | * kernel_text_address() verifies we are not in core kernel | ||
114 | * init code, see jump_label_invalidate_module_init(). | ||
115 | */ | ||
116 | if (entry->code && kernel_text_address(entry->code)) | ||
117 | arch_jump_label_transform(entry, enable); | ||
118 | } | ||
271 | } | 119 | } |
272 | 120 | ||
273 | /* | 121 | /* |
@@ -277,142 +125,173 @@ void __weak arch_jump_label_text_poke_early(jump_label_t addr) | |||
277 | { | 125 | { |
278 | } | 126 | } |
279 | 127 | ||
280 | static __init int init_jump_label(void) | 128 | static __init int jump_label_init(void) |
281 | { | 129 | { |
282 | int ret; | ||
283 | struct jump_entry *iter_start = __start___jump_table; | 130 | struct jump_entry *iter_start = __start___jump_table; |
284 | struct jump_entry *iter_stop = __stop___jump_table; | 131 | struct jump_entry *iter_stop = __stop___jump_table; |
132 | struct jump_label_key *key = NULL; | ||
285 | struct jump_entry *iter; | 133 | struct jump_entry *iter; |
286 | 134 | ||
287 | jump_label_lock(); | 135 | jump_label_lock(); |
288 | ret = build_jump_label_hashtable(__start___jump_table, | 136 | jump_label_sort_entries(iter_start, iter_stop); |
289 | __stop___jump_table); | 137 | |
290 | iter = iter_start; | 138 | for (iter = iter_start; iter < iter_stop; iter++) { |
291 | while (iter < iter_stop) { | ||
292 | arch_jump_label_text_poke_early(iter->code); | 139 | arch_jump_label_text_poke_early(iter->code); |
293 | iter++; | 140 | if (iter->key == (jump_label_t)(unsigned long)key) |
141 | continue; | ||
142 | |||
143 | key = (struct jump_label_key *)(unsigned long)iter->key; | ||
144 | atomic_set(&key->enabled, 0); | ||
145 | key->entries = iter; | ||
146 | #ifdef CONFIG_MODULES | ||
147 | key->next = NULL; | ||
148 | #endif | ||
294 | } | 149 | } |
295 | jump_label_unlock(); | 150 | jump_label_unlock(); |
296 | return ret; | 151 | |
152 | return 0; | ||
297 | } | 153 | } |
298 | early_initcall(init_jump_label); | 154 | early_initcall(jump_label_init); |
299 | 155 | ||
300 | #ifdef CONFIG_MODULES | 156 | #ifdef CONFIG_MODULES |
301 | 157 | ||
302 | static struct jump_label_module_entry * | 158 | struct jump_label_mod { |
303 | add_jump_label_module_entry(struct jump_label_entry *entry, | 159 | struct jump_label_mod *next; |
304 | struct jump_entry *iter_begin, | 160 | struct jump_entry *entries; |
305 | int count, struct module *mod) | 161 | struct module *mod; |
162 | }; | ||
163 | |||
164 | static int __jump_label_mod_text_reserved(void *start, void *end) | ||
165 | { | ||
166 | struct module *mod; | ||
167 | |||
168 | mod = __module_text_address((unsigned long)start); | ||
169 | if (!mod) | ||
170 | return 0; | ||
171 | |||
172 | WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); | ||
173 | |||
174 | return __jump_label_text_reserved(mod->jump_entries, | ||
175 | mod->jump_entries + mod->num_jump_entries, | ||
176 | start, end); | ||
177 | } | ||
178 | |||
179 | static void __jump_label_mod_update(struct jump_label_key *key, int enable) | ||
180 | { | ||
181 | struct jump_label_mod *mod = key->next; | ||
182 | |||
183 | while (mod) { | ||
184 | __jump_label_update(key, mod->entries, enable); | ||
185 | mod = mod->next; | ||
186 | } | ||
187 | } | ||
188 | |||
189 | /*** | ||
190 | * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() | ||
191 | * @mod: module to patch | ||
192 | * | ||
193 | * Allow for run-time selection of the optimal nops. Before the module | ||
194 | * loads patch these with arch_get_jump_label_nop(), which is specified by | ||
195 | * the arch specific jump label code. | ||
196 | */ | ||
197 | void jump_label_apply_nops(struct module *mod) | ||
306 | { | 198 | { |
307 | struct jump_label_module_entry *e; | 199 | struct jump_entry *iter_start = mod->jump_entries; |
308 | 200 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; | |
309 | e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL); | 201 | struct jump_entry *iter; |
310 | if (!e) | 202 | |
311 | return ERR_PTR(-ENOMEM); | 203 | /* if the module doesn't have jump label entries, just return */ |
312 | e->mod = mod; | 204 | if (iter_start == iter_stop) |
313 | e->nr_entries = count; | 205 | return; |
314 | e->table = iter_begin; | 206 | |
315 | hlist_add_head(&e->hlist, &entry->modules); | 207 | for (iter = iter_start; iter < iter_stop; iter++) |
316 | return e; | 208 | arch_jump_label_text_poke_early(iter->code); |
317 | } | 209 | } |
318 | 210 | ||
319 | static int add_jump_label_module(struct module *mod) | 211 | static int jump_label_add_module(struct module *mod) |
320 | { | 212 | { |
321 | struct jump_entry *iter, *iter_begin; | 213 | struct jump_entry *iter_start = mod->jump_entries; |
322 | struct jump_label_entry *entry; | 214 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; |
323 | struct jump_label_module_entry *module_entry; | 215 | struct jump_entry *iter; |
324 | int count; | 216 | struct jump_label_key *key = NULL; |
217 | struct jump_label_mod *jlm; | ||
325 | 218 | ||
326 | /* if the module doesn't have jump label entries, just return */ | 219 | /* if the module doesn't have jump label entries, just return */ |
327 | if (!mod->num_jump_entries) | 220 | if (iter_start == iter_stop) |
328 | return 0; | 221 | return 0; |
329 | 222 | ||
330 | sort_jump_label_entries(mod->jump_entries, | 223 | jump_label_sort_entries(iter_start, iter_stop); |
331 | mod->jump_entries + mod->num_jump_entries); | 224 | |
332 | iter = mod->jump_entries; | 225 | for (iter = iter_start; iter < iter_stop; iter++) { |
333 | while (iter < mod->jump_entries + mod->num_jump_entries) { | 226 | if (iter->key == (jump_label_t)(unsigned long)key) |
334 | entry = get_jump_label_entry(iter->key); | 227 | continue; |
335 | iter_begin = iter; | 228 | |
336 | count = 0; | 229 | key = (struct jump_label_key *)(unsigned long)iter->key; |
337 | while ((iter < mod->jump_entries + mod->num_jump_entries) && | 230 | |
338 | (iter->key == iter_begin->key)) { | 231 | if (__module_address(iter->key) == mod) { |
339 | iter++; | 232 | atomic_set(&key->enabled, 0); |
340 | count++; | 233 | key->entries = iter; |
341 | } | 234 | key->next = NULL; |
342 | if (!entry) { | 235 | continue; |
343 | entry = add_jump_label_entry(iter_begin->key, 0, NULL); | ||
344 | if (IS_ERR(entry)) | ||
345 | return PTR_ERR(entry); | ||
346 | } | 236 | } |
347 | module_entry = add_jump_label_module_entry(entry, iter_begin, | 237 | |
348 | count, mod); | 238 | jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL); |
349 | if (IS_ERR(module_entry)) | 239 | if (!jlm) |
350 | return PTR_ERR(module_entry); | 240 | return -ENOMEM; |
241 | |||
242 | jlm->mod = mod; | ||
243 | jlm->entries = iter; | ||
244 | jlm->next = key->next; | ||
245 | key->next = jlm; | ||
246 | |||
247 | if (jump_label_enabled(key)) | ||
248 | __jump_label_update(key, iter, JUMP_LABEL_ENABLE); | ||
351 | } | 249 | } |
250 | |||
352 | return 0; | 251 | return 0; |
353 | } | 252 | } |
354 | 253 | ||
355 | static void remove_jump_label_module(struct module *mod) | 254 | static void jump_label_del_module(struct module *mod) |
356 | { | 255 | { |
357 | struct hlist_head *head; | 256 | struct jump_entry *iter_start = mod->jump_entries; |
358 | struct hlist_node *node, *node_next, *module_node, *module_node_next; | 257 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; |
359 | struct jump_label_entry *e; | 258 | struct jump_entry *iter; |
360 | struct jump_label_module_entry *e_module; | 259 | struct jump_label_key *key = NULL; |
361 | int i; | 260 | struct jump_label_mod *jlm, **prev; |
362 | 261 | ||
363 | /* if the module doesn't have jump label entries, just return */ | 262 | for (iter = iter_start; iter < iter_stop; iter++) { |
364 | if (!mod->num_jump_entries) | 263 | if (iter->key == (jump_label_t)(unsigned long)key) |
365 | return; | 264 | continue; |
265 | |||
266 | key = (struct jump_label_key *)(unsigned long)iter->key; | ||
267 | |||
268 | if (__module_address(iter->key) == mod) | ||
269 | continue; | ||
270 | |||
271 | prev = &key->next; | ||
272 | jlm = key->next; | ||
366 | 273 | ||
367 | for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { | 274 | while (jlm && jlm->mod != mod) { |
368 | head = &jump_label_table[i]; | 275 | prev = &jlm->next; |
369 | hlist_for_each_entry_safe(e, node, node_next, head, hlist) { | 276 | jlm = jlm->next; |
370 | hlist_for_each_entry_safe(e_module, module_node, | 277 | } |
371 | module_node_next, | 278 | |
372 | &(e->modules), hlist) { | 279 | if (jlm) { |
373 | if (e_module->mod == mod) { | 280 | *prev = jlm->next; |
374 | hlist_del(&e_module->hlist); | 281 | kfree(jlm); |
375 | kfree(e_module); | ||
376 | } | ||
377 | } | ||
378 | if (hlist_empty(&e->modules) && (e->nr_entries == 0)) { | ||
379 | hlist_del(&e->hlist); | ||
380 | kfree(e); | ||
381 | } | ||
382 | } | 282 | } |
383 | } | 283 | } |
384 | } | 284 | } |
385 | 285 | ||
386 | static void remove_jump_label_module_init(struct module *mod) | 286 | static void jump_label_invalidate_module_init(struct module *mod) |
387 | { | 287 | { |
388 | struct hlist_head *head; | 288 | struct jump_entry *iter_start = mod->jump_entries; |
389 | struct hlist_node *node, *node_next, *module_node, *module_node_next; | 289 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; |
390 | struct jump_label_entry *e; | ||
391 | struct jump_label_module_entry *e_module; | ||
392 | struct jump_entry *iter; | 290 | struct jump_entry *iter; |
393 | int i, count; | ||
394 | |||
395 | /* if the module doesn't have jump label entries, just return */ | ||
396 | if (!mod->num_jump_entries) | ||
397 | return; | ||
398 | 291 | ||
399 | for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { | 292 | for (iter = iter_start; iter < iter_stop; iter++) { |
400 | head = &jump_label_table[i]; | 293 | if (within_module_init(iter->code, mod)) |
401 | hlist_for_each_entry_safe(e, node, node_next, head, hlist) { | 294 | iter->code = 0; |
402 | hlist_for_each_entry_safe(e_module, module_node, | ||
403 | module_node_next, | ||
404 | &(e->modules), hlist) { | ||
405 | if (e_module->mod != mod) | ||
406 | continue; | ||
407 | count = e_module->nr_entries; | ||
408 | iter = e_module->table; | ||
409 | while (count--) { | ||
410 | if (within_module_init(iter->code, mod)) | ||
411 | iter->key = 0; | ||
412 | iter++; | ||
413 | } | ||
414 | } | ||
415 | } | ||
416 | } | 295 | } |
417 | } | 296 | } |
418 | 297 | ||
@@ -426,59 +305,77 @@ jump_label_module_notify(struct notifier_block *self, unsigned long val, | |||
426 | switch (val) { | 305 | switch (val) { |
427 | case MODULE_STATE_COMING: | 306 | case MODULE_STATE_COMING: |
428 | jump_label_lock(); | 307 | jump_label_lock(); |
429 | ret = add_jump_label_module(mod); | 308 | ret = jump_label_add_module(mod); |
430 | if (ret) | 309 | if (ret) |
431 | remove_jump_label_module(mod); | 310 | jump_label_del_module(mod); |
432 | jump_label_unlock(); | 311 | jump_label_unlock(); |
433 | break; | 312 | break; |
434 | case MODULE_STATE_GOING: | 313 | case MODULE_STATE_GOING: |
435 | jump_label_lock(); | 314 | jump_label_lock(); |
436 | remove_jump_label_module(mod); | 315 | jump_label_del_module(mod); |
437 | jump_label_unlock(); | 316 | jump_label_unlock(); |
438 | break; | 317 | break; |
439 | case MODULE_STATE_LIVE: | 318 | case MODULE_STATE_LIVE: |
440 | jump_label_lock(); | 319 | jump_label_lock(); |
441 | remove_jump_label_module_init(mod); | 320 | jump_label_invalidate_module_init(mod); |
442 | jump_label_unlock(); | 321 | jump_label_unlock(); |
443 | break; | 322 | break; |
444 | } | 323 | } |
445 | return ret; | ||
446 | } | ||
447 | 324 | ||
448 | /*** | 325 | return notifier_from_errno(ret); |
449 | * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() | ||
450 | * @mod: module to patch | ||
451 | * | ||
452 | * Allow for run-time selection of the optimal nops. Before the module | ||
453 | * loads patch these with arch_get_jump_label_nop(), which is specified by | ||
454 | * the arch specific jump label code. | ||
455 | */ | ||
456 | void jump_label_apply_nops(struct module *mod) | ||
457 | { | ||
458 | struct jump_entry *iter; | ||
459 | |||
460 | /* if the module doesn't have jump label entries, just return */ | ||
461 | if (!mod->num_jump_entries) | ||
462 | return; | ||
463 | |||
464 | iter = mod->jump_entries; | ||
465 | while (iter < mod->jump_entries + mod->num_jump_entries) { | ||
466 | arch_jump_label_text_poke_early(iter->code); | ||
467 | iter++; | ||
468 | } | ||
469 | } | 326 | } |
470 | 327 | ||
471 | struct notifier_block jump_label_module_nb = { | 328 | struct notifier_block jump_label_module_nb = { |
472 | .notifier_call = jump_label_module_notify, | 329 | .notifier_call = jump_label_module_notify, |
473 | .priority = 0, | 330 | .priority = 1, /* higher than tracepoints */ |
474 | }; | 331 | }; |
475 | 332 | ||
476 | static __init int init_jump_label_module(void) | 333 | static __init int jump_label_init_module(void) |
477 | { | 334 | { |
478 | return register_module_notifier(&jump_label_module_nb); | 335 | return register_module_notifier(&jump_label_module_nb); |
479 | } | 336 | } |
480 | early_initcall(init_jump_label_module); | 337 | early_initcall(jump_label_init_module); |
481 | 338 | ||
482 | #endif /* CONFIG_MODULES */ | 339 | #endif /* CONFIG_MODULES */ |
483 | 340 | ||
341 | /*** | ||
342 | * jump_label_text_reserved - check if addr range is reserved | ||
343 | * @start: start text addr | ||
344 | * @end: end text addr | ||
345 | * | ||
346 | * checks if the text addr located between @start and @end | ||
347 | * overlaps with any of the jump label patch addresses. Code | ||
348 | * that wants to modify kernel text should first verify that | ||
349 | * it does not overlap with any of the jump label addresses. | ||
350 | * Caller must hold jump_label_mutex. | ||
351 | * | ||
352 | * returns 1 if there is an overlap, 0 otherwise | ||
353 | */ | ||
354 | int jump_label_text_reserved(void *start, void *end) | ||
355 | { | ||
356 | int ret = __jump_label_text_reserved(__start___jump_table, | ||
357 | __stop___jump_table, start, end); | ||
358 | |||
359 | if (ret) | ||
360 | return ret; | ||
361 | |||
362 | #ifdef CONFIG_MODULES | ||
363 | ret = __jump_label_mod_text_reserved(start, end); | ||
364 | #endif | ||
365 | return ret; | ||
366 | } | ||
367 | |||
368 | static void jump_label_update(struct jump_label_key *key, int enable) | ||
369 | { | ||
370 | struct jump_entry *entry = key->entries; | ||
371 | |||
372 | /* if there are no users, entry can be NULL */ | ||
373 | if (entry) | ||
374 | __jump_label_update(key, entry, enable); | ||
375 | |||
376 | #ifdef CONFIG_MODULES | ||
377 | __jump_label_mod_update(key, enable); | ||
378 | #endif | ||
379 | } | ||
380 | |||
484 | #endif | 381 | #endif |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index ee24fa1935ac..d017c2c82c44 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
@@ -39,20 +39,26 @@ | |||
39 | #include "trace_stat.h" | 39 | #include "trace_stat.h" |
40 | 40 | ||
41 | #define FTRACE_WARN_ON(cond) \ | 41 | #define FTRACE_WARN_ON(cond) \ |
42 | do { \ | 42 | ({ \ |
43 | if (WARN_ON(cond)) \ | 43 | int ___r = cond; \ |
44 | if (WARN_ON(___r)) \ | ||
44 | ftrace_kill(); \ | 45 | ftrace_kill(); \ |
45 | } while (0) | 46 | ___r; \ |
47 | }) | ||
46 | 48 | ||
47 | #define FTRACE_WARN_ON_ONCE(cond) \ | 49 | #define FTRACE_WARN_ON_ONCE(cond) \ |
48 | do { \ | 50 | ({ \ |
49 | if (WARN_ON_ONCE(cond)) \ | 51 | int ___r = cond; \ |
52 | if (WARN_ON_ONCE(___r)) \ | ||
50 | ftrace_kill(); \ | 53 | ftrace_kill(); \ |
51 | } while (0) | 54 | ___r; \ |
55 | }) | ||
52 | 56 | ||
53 | /* hash bits for specific function selection */ | 57 | /* hash bits for specific function selection */ |
54 | #define FTRACE_HASH_BITS 7 | 58 | #define FTRACE_HASH_BITS 7 |
55 | #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) | 59 | #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) |
60 | #define FTRACE_HASH_DEFAULT_BITS 10 | ||
61 | #define FTRACE_HASH_MAX_BITS 12 | ||
56 | 62 | ||
57 | /* ftrace_enabled is a method to turn ftrace on or off */ | 63 | /* ftrace_enabled is a method to turn ftrace on or off */ |
58 | int ftrace_enabled __read_mostly; | 64 | int ftrace_enabled __read_mostly; |
@@ -81,23 +87,29 @@ static struct ftrace_ops ftrace_list_end __read_mostly = | |||
81 | .func = ftrace_stub, | 87 | .func = ftrace_stub, |
82 | }; | 88 | }; |
83 | 89 | ||
84 | static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end; | 90 | static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end; |
91 | static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end; | ||
85 | ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; | 92 | ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; |
86 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; | 93 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; |
87 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; | 94 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; |
95 | static struct ftrace_ops global_ops; | ||
96 | |||
97 | static void | ||
98 | ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip); | ||
88 | 99 | ||
89 | /* | 100 | /* |
90 | * Traverse the ftrace_list, invoking all entries. The reason that we | 101 | * Traverse the ftrace_global_list, invoking all entries. The reason that we |
91 | * can use rcu_dereference_raw() is that elements removed from this list | 102 | * can use rcu_dereference_raw() is that elements removed from this list |
92 | * are simply leaked, so there is no need to interact with a grace-period | 103 | * are simply leaked, so there is no need to interact with a grace-period |
93 | * mechanism. The rcu_dereference_raw() calls are needed to handle | 104 | * mechanism. The rcu_dereference_raw() calls are needed to handle |
94 | * concurrent insertions into the ftrace_list. | 105 | * concurrent insertions into the ftrace_global_list. |
95 | * | 106 | * |
96 | * Silly Alpha and silly pointer-speculation compiler optimizations! | 107 | * Silly Alpha and silly pointer-speculation compiler optimizations! |
97 | */ | 108 | */ |
98 | static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) | 109 | static void ftrace_global_list_func(unsigned long ip, |
110 | unsigned long parent_ip) | ||
99 | { | 111 | { |
100 | struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/ | 112 | struct ftrace_ops *op = rcu_dereference_raw(ftrace_global_list); /*see above*/ |
101 | 113 | ||
102 | while (op != &ftrace_list_end) { | 114 | while (op != &ftrace_list_end) { |
103 | op->func(ip, parent_ip); | 115 | op->func(ip, parent_ip); |
@@ -147,46 +159,69 @@ static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip) | |||
147 | } | 159 | } |
148 | #endif | 160 | #endif |
149 | 161 | ||
150 | static int __register_ftrace_function(struct ftrace_ops *ops) | 162 | static void update_global_ops(void) |
151 | { | 163 | { |
152 | ops->next = ftrace_list; | 164 | ftrace_func_t func; |
165 | |||
153 | /* | 166 | /* |
154 | * We are entering ops into the ftrace_list but another | 167 | * If there's only one function registered, then call that |
155 | * CPU might be walking that list. We need to make sure | 168 | * function directly. Otherwise, we need to iterate over the |
156 | * the ops->next pointer is valid before another CPU sees | 169 | * registered callers. |
157 | * the ops pointer included into the ftrace_list. | ||
158 | */ | 170 | */ |
159 | rcu_assign_pointer(ftrace_list, ops); | 171 | if (ftrace_global_list == &ftrace_list_end || |
172 | ftrace_global_list->next == &ftrace_list_end) | ||
173 | func = ftrace_global_list->func; | ||
174 | else | ||
175 | func = ftrace_global_list_func; | ||
160 | 176 | ||
161 | if (ftrace_enabled) { | 177 | /* If we filter on pids, update to use the pid function */ |
162 | ftrace_func_t func; | 178 | if (!list_empty(&ftrace_pids)) { |
179 | set_ftrace_pid_function(func); | ||
180 | func = ftrace_pid_func; | ||
181 | } | ||
163 | 182 | ||
164 | if (ops->next == &ftrace_list_end) | 183 | global_ops.func = func; |
165 | func = ops->func; | 184 | } |
166 | else | ||
167 | func = ftrace_list_func; | ||
168 | 185 | ||
169 | if (!list_empty(&ftrace_pids)) { | 186 | static void update_ftrace_function(void) |
170 | set_ftrace_pid_function(func); | 187 | { |
171 | func = ftrace_pid_func; | 188 | ftrace_func_t func; |
172 | } | 189 | |
190 | update_global_ops(); | ||
191 | |||
192 | /* | ||
193 | * If we are at the end of the list and this ops is | ||
194 | * not dynamic, then have the mcount trampoline call | ||
195 | * the function directly | ||
196 | */ | ||
197 | if (ftrace_ops_list == &ftrace_list_end || | ||
198 | (ftrace_ops_list->next == &ftrace_list_end && | ||
199 | !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC))) | ||
200 | func = ftrace_ops_list->func; | ||
201 | else | ||
202 | func = ftrace_ops_list_func; | ||
173 | 203 | ||
174 | /* | ||
175 | * For one func, simply call it directly. | ||
176 | * For more than one func, call the chain. | ||
177 | */ | ||
178 | #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST | 204 | #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST |
179 | ftrace_trace_function = func; | 205 | ftrace_trace_function = func; |
180 | #else | 206 | #else |
181 | __ftrace_trace_function = func; | 207 | __ftrace_trace_function = func; |
182 | ftrace_trace_function = ftrace_test_stop_func; | 208 | ftrace_trace_function = ftrace_test_stop_func; |
183 | #endif | 209 | #endif |
184 | } | 210 | } |
185 | 211 | ||
186 | return 0; | 212 | static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) |
213 | { | ||
214 | ops->next = *list; | ||
215 | /* | ||
216 | * We are entering ops into the list but another | ||
217 | * CPU might be walking that list. We need to make sure | ||
218 | * the ops->next pointer is valid before another CPU sees | ||
219 | * the ops pointer included into the list. | ||
220 | */ | ||
221 | rcu_assign_pointer(*list, ops); | ||
187 | } | 222 | } |
188 | 223 | ||
189 | static int __unregister_ftrace_function(struct ftrace_ops *ops) | 224 | static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) |
190 | { | 225 | { |
191 | struct ftrace_ops **p; | 226 | struct ftrace_ops **p; |
192 | 227 | ||
@@ -194,13 +229,12 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops) | |||
194 | * If we are removing the last function, then simply point | 229 | * If we are removing the last function, then simply point |
195 | * to the ftrace_stub. | 230 | * to the ftrace_stub. |
196 | */ | 231 | */ |
197 | if (ftrace_list == ops && ops->next == &ftrace_list_end) { | 232 | if (*list == ops && ops->next == &ftrace_list_end) { |
198 | ftrace_trace_function = ftrace_stub; | 233 | *list = &ftrace_list_end; |
199 | ftrace_list = &ftrace_list_end; | ||
200 | return 0; | 234 | return 0; |
201 | } | 235 | } |
202 | 236 | ||
203 | for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next) | 237 | for (p = list; *p != &ftrace_list_end; p = &(*p)->next) |
204 | if (*p == ops) | 238 | if (*p == ops) |
205 | break; | 239 | break; |
206 | 240 | ||
@@ -208,53 +242,83 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops) | |||
208 | return -1; | 242 | return -1; |
209 | 243 | ||
210 | *p = (*p)->next; | 244 | *p = (*p)->next; |
245 | return 0; | ||
246 | } | ||
211 | 247 | ||
212 | if (ftrace_enabled) { | 248 | static int __register_ftrace_function(struct ftrace_ops *ops) |
213 | /* If we only have one func left, then call that directly */ | 249 | { |
214 | if (ftrace_list->next == &ftrace_list_end) { | 250 | if (ftrace_disabled) |
215 | ftrace_func_t func = ftrace_list->func; | 251 | return -ENODEV; |
216 | 252 | ||
217 | if (!list_empty(&ftrace_pids)) { | 253 | if (FTRACE_WARN_ON(ops == &global_ops)) |
218 | set_ftrace_pid_function(func); | 254 | return -EINVAL; |
219 | func = ftrace_pid_func; | 255 | |
220 | } | 256 | if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) |
221 | #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST | 257 | return -EBUSY; |
222 | ftrace_trace_function = func; | 258 | |
223 | #else | 259 | if (!core_kernel_data((unsigned long)ops)) |
224 | __ftrace_trace_function = func; | 260 | ops->flags |= FTRACE_OPS_FL_DYNAMIC; |
225 | #endif | 261 | |
226 | } | 262 | if (ops->flags & FTRACE_OPS_FL_GLOBAL) { |
227 | } | 263 | int first = ftrace_global_list == &ftrace_list_end; |
264 | add_ftrace_ops(&ftrace_global_list, ops); | ||
265 | ops->flags |= FTRACE_OPS_FL_ENABLED; | ||
266 | if (first) | ||
267 | add_ftrace_ops(&ftrace_ops_list, &global_ops); | ||
268 | } else | ||
269 | add_ftrace_ops(&ftrace_ops_list, ops); | ||
270 | |||
271 | if (ftrace_enabled) | ||
272 | update_ftrace_function(); | ||
228 | 273 | ||
229 | return 0; | 274 | return 0; |
230 | } | 275 | } |
231 | 276 | ||
232 | static void ftrace_update_pid_func(void) | 277 | static int __unregister_ftrace_function(struct ftrace_ops *ops) |
233 | { | 278 | { |
234 | ftrace_func_t func; | 279 | int ret; |
235 | 280 | ||
236 | if (ftrace_trace_function == ftrace_stub) | 281 | if (ftrace_disabled) |
237 | return; | 282 | return -ENODEV; |
238 | 283 | ||
239 | #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST | 284 | if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) |
240 | func = ftrace_trace_function; | 285 | return -EBUSY; |
241 | #else | ||
242 | func = __ftrace_trace_function; | ||
243 | #endif | ||
244 | 286 | ||
245 | if (!list_empty(&ftrace_pids)) { | 287 | if (FTRACE_WARN_ON(ops == &global_ops)) |
246 | set_ftrace_pid_function(func); | 288 | return -EINVAL; |
247 | func = ftrace_pid_func; | ||
248 | } else { | ||
249 | if (func == ftrace_pid_func) | ||
250 | func = ftrace_pid_function; | ||
251 | } | ||
252 | 289 | ||
253 | #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST | 290 | if (ops->flags & FTRACE_OPS_FL_GLOBAL) { |
254 | ftrace_trace_function = func; | 291 | ret = remove_ftrace_ops(&ftrace_global_list, ops); |
255 | #else | 292 | if (!ret && ftrace_global_list == &ftrace_list_end) |
256 | __ftrace_trace_function = func; | 293 | ret = remove_ftrace_ops(&ftrace_ops_list, &global_ops); |
257 | #endif | 294 | if (!ret) |
295 | ops->flags &= ~FTRACE_OPS_FL_ENABLED; | ||
296 | } else | ||
297 | ret = remove_ftrace_ops(&ftrace_ops_list, ops); | ||
298 | |||
299 | if (ret < 0) | ||
300 | return ret; | ||
301 | |||
302 | if (ftrace_enabled) | ||
303 | update_ftrace_function(); | ||
304 | |||
305 | /* | ||
306 | * Dynamic ops may be freed, we must make sure that all | ||
307 | * callers are done before leaving this function. | ||
308 | */ | ||
309 | if (ops->flags & FTRACE_OPS_FL_DYNAMIC) | ||
310 | synchronize_sched(); | ||
311 | |||
312 | return 0; | ||
313 | } | ||
314 | |||
315 | static void ftrace_update_pid_func(void) | ||
316 | { | ||
317 | /* Only do something if we are tracing something */ | ||
318 | if (ftrace_trace_function == ftrace_stub) | ||
319 | return; | ||
320 | |||
321 | update_ftrace_function(); | ||
258 | } | 322 | } |
259 | 323 | ||
260 | #ifdef CONFIG_FUNCTION_PROFILER | 324 | #ifdef CONFIG_FUNCTION_PROFILER |
@@ -888,8 +952,35 @@ enum { | |||
888 | FTRACE_START_FUNC_RET = (1 << 3), | 952 | FTRACE_START_FUNC_RET = (1 << 3), |
889 | FTRACE_STOP_FUNC_RET = (1 << 4), | 953 | FTRACE_STOP_FUNC_RET = (1 << 4), |
890 | }; | 954 | }; |
955 | struct ftrace_func_entry { | ||
956 | struct hlist_node hlist; | ||
957 | unsigned long ip; | ||
958 | }; | ||
891 | 959 | ||
892 | static int ftrace_filtered; | 960 | struct ftrace_hash { |
961 | unsigned long size_bits; | ||
962 | struct hlist_head *buckets; | ||
963 | unsigned long count; | ||
964 | struct rcu_head rcu; | ||
965 | }; | ||
966 | |||
967 | /* | ||
968 | * We make these constant because no one should touch them, | ||
969 | * but they are used as the default "empty hash", to avoid allocating | ||
970 | * it all the time. These are in a read only section such that if | ||
971 | * anyone does try to modify it, it will cause an exception. | ||
972 | */ | ||
973 | static const struct hlist_head empty_buckets[1]; | ||
974 | static const struct ftrace_hash empty_hash = { | ||
975 | .buckets = (struct hlist_head *)empty_buckets, | ||
976 | }; | ||
977 | #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) | ||
978 | |||
979 | static struct ftrace_ops global_ops = { | ||
980 | .func = ftrace_stub, | ||
981 | .notrace_hash = EMPTY_HASH, | ||
982 | .filter_hash = EMPTY_HASH, | ||
983 | }; | ||
893 | 984 | ||
894 | static struct dyn_ftrace *ftrace_new_addrs; | 985 | static struct dyn_ftrace *ftrace_new_addrs; |
895 | 986 | ||
@@ -912,6 +1003,269 @@ static struct ftrace_page *ftrace_pages; | |||
912 | 1003 | ||
913 | static struct dyn_ftrace *ftrace_free_records; | 1004 | static struct dyn_ftrace *ftrace_free_records; |
914 | 1005 | ||
1006 | static struct ftrace_func_entry * | ||
1007 | ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) | ||
1008 | { | ||
1009 | unsigned long key; | ||
1010 | struct ftrace_func_entry *entry; | ||
1011 | struct hlist_head *hhd; | ||
1012 | struct hlist_node *n; | ||
1013 | |||
1014 | if (!hash->count) | ||
1015 | return NULL; | ||
1016 | |||
1017 | if (hash->size_bits > 0) | ||
1018 | key = hash_long(ip, hash->size_bits); | ||
1019 | else | ||
1020 | key = 0; | ||
1021 | |||
1022 | hhd = &hash->buckets[key]; | ||
1023 | |||
1024 | hlist_for_each_entry_rcu(entry, n, hhd, hlist) { | ||
1025 | if (entry->ip == ip) | ||
1026 | return entry; | ||
1027 | } | ||
1028 | return NULL; | ||
1029 | } | ||
1030 | |||
1031 | static void __add_hash_entry(struct ftrace_hash *hash, | ||
1032 | struct ftrace_func_entry *entry) | ||
1033 | { | ||
1034 | struct hlist_head *hhd; | ||
1035 | unsigned long key; | ||
1036 | |||
1037 | if (hash->size_bits) | ||
1038 | key = hash_long(entry->ip, hash->size_bits); | ||
1039 | else | ||
1040 | key = 0; | ||
1041 | |||
1042 | hhd = &hash->buckets[key]; | ||
1043 | hlist_add_head(&entry->hlist, hhd); | ||
1044 | hash->count++; | ||
1045 | } | ||
1046 | |||
1047 | static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip) | ||
1048 | { | ||
1049 | struct ftrace_func_entry *entry; | ||
1050 | |||
1051 | entry = kmalloc(sizeof(*entry), GFP_KERNEL); | ||
1052 | if (!entry) | ||
1053 | return -ENOMEM; | ||
1054 | |||
1055 | entry->ip = ip; | ||
1056 | __add_hash_entry(hash, entry); | ||
1057 | |||
1058 | return 0; | ||
1059 | } | ||
1060 | |||
1061 | static void | ||
1062 | free_hash_entry(struct ftrace_hash *hash, | ||
1063 | struct ftrace_func_entry *entry) | ||
1064 | { | ||
1065 | hlist_del(&entry->hlist); | ||
1066 | kfree(entry); | ||
1067 | hash->count--; | ||
1068 | } | ||
1069 | |||
1070 | static void | ||
1071 | remove_hash_entry(struct ftrace_hash *hash, | ||
1072 | struct ftrace_func_entry *entry) | ||
1073 | { | ||
1074 | hlist_del(&entry->hlist); | ||
1075 | hash->count--; | ||
1076 | } | ||
1077 | |||
1078 | static void ftrace_hash_clear(struct ftrace_hash *hash) | ||
1079 | { | ||
1080 | struct hlist_head *hhd; | ||
1081 | struct hlist_node *tp, *tn; | ||
1082 | struct ftrace_func_entry *entry; | ||
1083 | int size = 1 << hash->size_bits; | ||
1084 | int i; | ||
1085 | |||
1086 | if (!hash->count) | ||
1087 | return; | ||
1088 | |||
1089 | for (i = 0; i < size; i++) { | ||
1090 | hhd = &hash->buckets[i]; | ||
1091 | hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) | ||
1092 | free_hash_entry(hash, entry); | ||
1093 | } | ||
1094 | FTRACE_WARN_ON(hash->count); | ||
1095 | } | ||
1096 | |||
1097 | static void free_ftrace_hash(struct ftrace_hash *hash) | ||
1098 | { | ||
1099 | if (!hash || hash == EMPTY_HASH) | ||
1100 | return; | ||
1101 | ftrace_hash_clear(hash); | ||
1102 | kfree(hash->buckets); | ||
1103 | kfree(hash); | ||
1104 | } | ||
1105 | |||
1106 | static void __free_ftrace_hash_rcu(struct rcu_head *rcu) | ||
1107 | { | ||
1108 | struct ftrace_hash *hash; | ||
1109 | |||
1110 | hash = container_of(rcu, struct ftrace_hash, rcu); | ||
1111 | free_ftrace_hash(hash); | ||
1112 | } | ||
1113 | |||
1114 | static void free_ftrace_hash_rcu(struct ftrace_hash *hash) | ||
1115 | { | ||
1116 | if (!hash || hash == EMPTY_HASH) | ||
1117 | return; | ||
1118 | call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu); | ||
1119 | } | ||
1120 | |||
1121 | static struct ftrace_hash *alloc_ftrace_hash(int size_bits) | ||
1122 | { | ||
1123 | struct ftrace_hash *hash; | ||
1124 | int size; | ||
1125 | |||
1126 | hash = kzalloc(sizeof(*hash), GFP_KERNEL); | ||
1127 | if (!hash) | ||
1128 | return NULL; | ||
1129 | |||
1130 | size = 1 << size_bits; | ||
1131 | hash->buckets = kzalloc(sizeof(*hash->buckets) * size, GFP_KERNEL); | ||
1132 | |||
1133 | if (!hash->buckets) { | ||
1134 | kfree(hash); | ||
1135 | return NULL; | ||
1136 | } | ||
1137 | |||
1138 | hash->size_bits = size_bits; | ||
1139 | |||
1140 | return hash; | ||
1141 | } | ||
1142 | |||
1143 | static struct ftrace_hash * | ||
1144 | alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) | ||
1145 | { | ||
1146 | struct ftrace_func_entry *entry; | ||
1147 | struct ftrace_hash *new_hash; | ||
1148 | struct hlist_node *tp; | ||
1149 | int size; | ||
1150 | int ret; | ||
1151 | int i; | ||
1152 | |||
1153 | new_hash = alloc_ftrace_hash(size_bits); | ||
1154 | if (!new_hash) | ||
1155 | return NULL; | ||
1156 | |||
1157 | /* Empty hash? */ | ||
1158 | if (!hash || !hash->count) | ||
1159 | return new_hash; | ||
1160 | |||
1161 | size = 1 << hash->size_bits; | ||
1162 | for (i = 0; i < size; i++) { | ||
1163 | hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) { | ||
1164 | ret = add_hash_entry(new_hash, entry->ip); | ||
1165 | if (ret < 0) | ||
1166 | goto free_hash; | ||
1167 | } | ||
1168 | } | ||
1169 | |||
1170 | FTRACE_WARN_ON(new_hash->count != hash->count); | ||
1171 | |||
1172 | return new_hash; | ||
1173 | |||
1174 | free_hash: | ||
1175 | free_ftrace_hash(new_hash); | ||
1176 | return NULL; | ||
1177 | } | ||
1178 | |||
1179 | static int | ||
1180 | ftrace_hash_move(struct ftrace_hash **dst, struct ftrace_hash *src) | ||
1181 | { | ||
1182 | struct ftrace_func_entry *entry; | ||
1183 | struct hlist_node *tp, *tn; | ||
1184 | struct hlist_head *hhd; | ||
1185 | struct ftrace_hash *old_hash; | ||
1186 | struct ftrace_hash *new_hash; | ||
1187 | unsigned long key; | ||
1188 | int size = src->count; | ||
1189 | int bits = 0; | ||
1190 | int i; | ||
1191 | |||
1192 | /* | ||
1193 | * If the new source is empty, just free dst and assign it | ||
1194 | * the empty_hash. | ||
1195 | */ | ||
1196 | if (!src->count) { | ||
1197 | free_ftrace_hash_rcu(*dst); | ||
1198 | rcu_assign_pointer(*dst, EMPTY_HASH); | ||
1199 | return 0; | ||
1200 | } | ||
1201 | |||
1202 | /* | ||
1203 | * Make the hash size about 1/2 the # found | ||
1204 | */ | ||
1205 | for (size /= 2; size; size >>= 1) | ||
1206 | bits++; | ||
1207 | |||
1208 | /* Don't allocate too much */ | ||
1209 | if (bits > FTRACE_HASH_MAX_BITS) | ||
1210 | bits = FTRACE_HASH_MAX_BITS; | ||
1211 | |||
1212 | new_hash = alloc_ftrace_hash(bits); | ||
1213 | if (!new_hash) | ||
1214 | return -ENOMEM; | ||
1215 | |||
1216 | size = 1 << src->size_bits; | ||
1217 | for (i = 0; i < size; i++) { | ||
1218 | hhd = &src->buckets[i]; | ||
1219 | hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) { | ||
1220 | if (bits > 0) | ||
1221 | key = hash_long(entry->ip, bits); | ||
1222 | else | ||
1223 | key = 0; | ||
1224 | remove_hash_entry(src, entry); | ||
1225 | __add_hash_entry(new_hash, entry); | ||
1226 | } | ||
1227 | } | ||
1228 | |||
1229 | old_hash = *dst; | ||
1230 | rcu_assign_pointer(*dst, new_hash); | ||
1231 | free_ftrace_hash_rcu(old_hash); | ||
1232 | |||
1233 | return 0; | ||
1234 | } | ||
1235 | |||
1236 | /* | ||
1237 | * Test the hashes for this ops to see if we want to call | ||
1238 | * the ops->func or not. | ||
1239 | * | ||
1240 | * It's a match if the ip is in the ops->filter_hash or | ||
1241 | * the filter_hash does not exist or is empty, | ||
1242 | * AND | ||
1243 | * the ip is not in the ops->notrace_hash. | ||
1244 | * | ||
1245 | * This needs to be called with preemption disabled as | ||
1246 | * the hashes are freed with call_rcu_sched(). | ||
1247 | */ | ||
1248 | static int | ||
1249 | ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip) | ||
1250 | { | ||
1251 | struct ftrace_hash *filter_hash; | ||
1252 | struct ftrace_hash *notrace_hash; | ||
1253 | int ret; | ||
1254 | |||
1255 | filter_hash = rcu_dereference_raw(ops->filter_hash); | ||
1256 | notrace_hash = rcu_dereference_raw(ops->notrace_hash); | ||
1257 | |||
1258 | if ((!filter_hash || !filter_hash->count || | ||
1259 | ftrace_lookup_ip(filter_hash, ip)) && | ||
1260 | (!notrace_hash || !notrace_hash->count || | ||
1261 | !ftrace_lookup_ip(notrace_hash, ip))) | ||
1262 | ret = 1; | ||
1263 | else | ||
1264 | ret = 0; | ||
1265 | |||
1266 | return ret; | ||
1267 | } | ||
1268 | |||
915 | /* | 1269 | /* |
916 | * This is a double for. Do not use 'break' to break out of the loop, | 1270 | * This is a double for. Do not use 'break' to break out of the loop, |
917 | * you must use a goto. | 1271 | * you must use a goto. |
@@ -926,6 +1280,105 @@ static struct dyn_ftrace *ftrace_free_records; | |||
926 | } \ | 1280 | } \ |
927 | } | 1281 | } |
928 | 1282 | ||
1283 | static void __ftrace_hash_rec_update(struct ftrace_ops *ops, | ||
1284 | int filter_hash, | ||
1285 | bool inc) | ||
1286 | { | ||
1287 | struct ftrace_hash *hash; | ||
1288 | struct ftrace_hash *other_hash; | ||
1289 | struct ftrace_page *pg; | ||
1290 | struct dyn_ftrace *rec; | ||
1291 | int count = 0; | ||
1292 | int all = 0; | ||
1293 | |||
1294 | /* Only update if the ops has been registered */ | ||
1295 | if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) | ||
1296 | return; | ||
1297 | |||
1298 | /* | ||
1299 | * In the filter_hash case: | ||
1300 | * If the count is zero, we update all records. | ||
1301 | * Otherwise we just update the items in the hash. | ||
1302 | * | ||
1303 | * In the notrace_hash case: | ||
1304 | * We enable the update in the hash. | ||
1305 | * As disabling notrace means enabling the tracing, | ||
1306 | * and enabling notrace means disabling, the inc variable | ||
1307 | * gets inversed. | ||
1308 | */ | ||
1309 | if (filter_hash) { | ||
1310 | hash = ops->filter_hash; | ||
1311 | other_hash = ops->notrace_hash; | ||
1312 | if (!hash || !hash->count) | ||
1313 | all = 1; | ||
1314 | } else { | ||
1315 | inc = !inc; | ||
1316 | hash = ops->notrace_hash; | ||
1317 | other_hash = ops->filter_hash; | ||
1318 | /* | ||
1319 | * If the notrace hash has no items, | ||
1320 | * then there's nothing to do. | ||
1321 | */ | ||
1322 | if (hash && !hash->count) | ||
1323 | return; | ||
1324 | } | ||
1325 | |||
1326 | do_for_each_ftrace_rec(pg, rec) { | ||
1327 | int in_other_hash = 0; | ||
1328 | int in_hash = 0; | ||
1329 | int match = 0; | ||
1330 | |||
1331 | if (all) { | ||
1332 | /* | ||
1333 | * Only the filter_hash affects all records. | ||
1334 | * Update if the record is not in the notrace hash. | ||
1335 | */ | ||
1336 | if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) | ||
1337 | match = 1; | ||
1338 | } else { | ||
1339 | in_hash = hash && !!ftrace_lookup_ip(hash, rec->ip); | ||
1340 | in_other_hash = other_hash && !!ftrace_lookup_ip(other_hash, rec->ip); | ||
1341 | |||
1342 | /* | ||
1343 | * | ||
1344 | */ | ||
1345 | if (filter_hash && in_hash && !in_other_hash) | ||
1346 | match = 1; | ||
1347 | else if (!filter_hash && in_hash && | ||
1348 | (in_other_hash || !other_hash->count)) | ||
1349 | match = 1; | ||
1350 | } | ||
1351 | if (!match) | ||
1352 | continue; | ||
1353 | |||
1354 | if (inc) { | ||
1355 | rec->flags++; | ||
1356 | if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX)) | ||
1357 | return; | ||
1358 | } else { | ||
1359 | if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0)) | ||
1360 | return; | ||
1361 | rec->flags--; | ||
1362 | } | ||
1363 | count++; | ||
1364 | /* Shortcut, if we handled all records, we are done. */ | ||
1365 | if (!all && count == hash->count) | ||
1366 | return; | ||
1367 | } while_for_each_ftrace_rec(); | ||
1368 | } | ||
1369 | |||
1370 | static void ftrace_hash_rec_disable(struct ftrace_ops *ops, | ||
1371 | int filter_hash) | ||
1372 | { | ||
1373 | __ftrace_hash_rec_update(ops, filter_hash, 0); | ||
1374 | } | ||
1375 | |||
1376 | static void ftrace_hash_rec_enable(struct ftrace_ops *ops, | ||
1377 | int filter_hash) | ||
1378 | { | ||
1379 | __ftrace_hash_rec_update(ops, filter_hash, 1); | ||
1380 | } | ||
1381 | |||
929 | static void ftrace_free_rec(struct dyn_ftrace *rec) | 1382 | static void ftrace_free_rec(struct dyn_ftrace *rec) |
930 | { | 1383 | { |
931 | rec->freelist = ftrace_free_records; | 1384 | rec->freelist = ftrace_free_records; |
@@ -1047,18 +1500,18 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable) | |||
1047 | ftrace_addr = (unsigned long)FTRACE_ADDR; | 1500 | ftrace_addr = (unsigned long)FTRACE_ADDR; |
1048 | 1501 | ||
1049 | /* | 1502 | /* |
1050 | * If this record is not to be traced or we want to disable it, | 1503 | * If we are enabling tracing: |
1051 | * then disable it. | 1504 | * |
1505 | * If the record has a ref count, then we need to enable it | ||
1506 | * because someone is using it. | ||
1052 | * | 1507 | * |
1053 | * If we want to enable it and filtering is off, then enable it. | 1508 | * Otherwise we make sure its disabled. |
1054 | * | 1509 | * |
1055 | * If we want to enable it and filtering is on, enable it only if | 1510 | * If we are disabling tracing, then disable all records that |
1056 | * it's filtered | 1511 | * are enabled. |
1057 | */ | 1512 | */ |
1058 | if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) { | 1513 | if (enable && (rec->flags & ~FTRACE_FL_MASK)) |
1059 | if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER)) | 1514 | flag = FTRACE_FL_ENABLED; |
1060 | flag = FTRACE_FL_ENABLED; | ||
1061 | } | ||
1062 | 1515 | ||
1063 | /* If the state of this record hasn't changed, then do nothing */ | 1516 | /* If the state of this record hasn't changed, then do nothing */ |
1064 | if ((rec->flags & FTRACE_FL_ENABLED) == flag) | 1517 | if ((rec->flags & FTRACE_FL_ENABLED) == flag) |
@@ -1079,19 +1532,16 @@ static void ftrace_replace_code(int enable) | |||
1079 | struct ftrace_page *pg; | 1532 | struct ftrace_page *pg; |
1080 | int failed; | 1533 | int failed; |
1081 | 1534 | ||
1535 | if (unlikely(ftrace_disabled)) | ||
1536 | return; | ||
1537 | |||
1082 | do_for_each_ftrace_rec(pg, rec) { | 1538 | do_for_each_ftrace_rec(pg, rec) { |
1083 | /* | 1539 | /* Skip over free records */ |
1084 | * Skip over free records, records that have | 1540 | if (rec->flags & FTRACE_FL_FREE) |
1085 | * failed and not converted. | ||
1086 | */ | ||
1087 | if (rec->flags & FTRACE_FL_FREE || | ||
1088 | rec->flags & FTRACE_FL_FAILED || | ||
1089 | !(rec->flags & FTRACE_FL_CONVERTED)) | ||
1090 | continue; | 1541 | continue; |
1091 | 1542 | ||
1092 | failed = __ftrace_replace_code(rec, enable); | 1543 | failed = __ftrace_replace_code(rec, enable); |
1093 | if (failed) { | 1544 | if (failed) { |
1094 | rec->flags |= FTRACE_FL_FAILED; | ||
1095 | ftrace_bug(failed, rec->ip); | 1545 | ftrace_bug(failed, rec->ip); |
1096 | /* Stop processing */ | 1546 | /* Stop processing */ |
1097 | return; | 1547 | return; |
@@ -1107,10 +1557,12 @@ ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) | |||
1107 | 1557 | ||
1108 | ip = rec->ip; | 1558 | ip = rec->ip; |
1109 | 1559 | ||
1560 | if (unlikely(ftrace_disabled)) | ||
1561 | return 0; | ||
1562 | |||
1110 | ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); | 1563 | ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); |
1111 | if (ret) { | 1564 | if (ret) { |
1112 | ftrace_bug(ret, ip); | 1565 | ftrace_bug(ret, ip); |
1113 | rec->flags |= FTRACE_FL_FAILED; | ||
1114 | return 0; | 1566 | return 0; |
1115 | } | 1567 | } |
1116 | return 1; | 1568 | return 1; |
@@ -1171,6 +1623,7 @@ static void ftrace_run_update_code(int command) | |||
1171 | 1623 | ||
1172 | static ftrace_func_t saved_ftrace_func; | 1624 | static ftrace_func_t saved_ftrace_func; |
1173 | static int ftrace_start_up; | 1625 | static int ftrace_start_up; |
1626 | static int global_start_up; | ||
1174 | 1627 | ||
1175 | static void ftrace_startup_enable(int command) | 1628 | static void ftrace_startup_enable(int command) |
1176 | { | 1629 | { |
@@ -1185,19 +1638,36 @@ static void ftrace_startup_enable(int command) | |||
1185 | ftrace_run_update_code(command); | 1638 | ftrace_run_update_code(command); |
1186 | } | 1639 | } |
1187 | 1640 | ||
1188 | static void ftrace_startup(int command) | 1641 | static void ftrace_startup(struct ftrace_ops *ops, int command) |
1189 | { | 1642 | { |
1643 | bool hash_enable = true; | ||
1644 | |||
1190 | if (unlikely(ftrace_disabled)) | 1645 | if (unlikely(ftrace_disabled)) |
1191 | return; | 1646 | return; |
1192 | 1647 | ||
1193 | ftrace_start_up++; | 1648 | ftrace_start_up++; |
1194 | command |= FTRACE_ENABLE_CALLS; | 1649 | command |= FTRACE_ENABLE_CALLS; |
1195 | 1650 | ||
1651 | /* ops marked global share the filter hashes */ | ||
1652 | if (ops->flags & FTRACE_OPS_FL_GLOBAL) { | ||
1653 | ops = &global_ops; | ||
1654 | /* Don't update hash if global is already set */ | ||
1655 | if (global_start_up) | ||
1656 | hash_enable = false; | ||
1657 | global_start_up++; | ||
1658 | } | ||
1659 | |||
1660 | ops->flags |= FTRACE_OPS_FL_ENABLED; | ||
1661 | if (hash_enable) | ||
1662 | ftrace_hash_rec_enable(ops, 1); | ||
1663 | |||
1196 | ftrace_startup_enable(command); | 1664 | ftrace_startup_enable(command); |
1197 | } | 1665 | } |
1198 | 1666 | ||
1199 | static void ftrace_shutdown(int command) | 1667 | static void ftrace_shutdown(struct ftrace_ops *ops, int command) |
1200 | { | 1668 | { |
1669 | bool hash_disable = true; | ||
1670 | |||
1201 | if (unlikely(ftrace_disabled)) | 1671 | if (unlikely(ftrace_disabled)) |
1202 | return; | 1672 | return; |
1203 | 1673 | ||
@@ -1209,6 +1679,23 @@ static void ftrace_shutdown(int command) | |||
1209 | */ | 1679 | */ |
1210 | WARN_ON_ONCE(ftrace_start_up < 0); | 1680 | WARN_ON_ONCE(ftrace_start_up < 0); |
1211 | 1681 | ||
1682 | if (ops->flags & FTRACE_OPS_FL_GLOBAL) { | ||
1683 | ops = &global_ops; | ||
1684 | global_start_up--; | ||
1685 | WARN_ON_ONCE(global_start_up < 0); | ||
1686 | /* Don't update hash if global still has users */ | ||
1687 | if (global_start_up) { | ||
1688 | WARN_ON_ONCE(!ftrace_start_up); | ||
1689 | hash_disable = false; | ||
1690 | } | ||
1691 | } | ||
1692 | |||
1693 | if (hash_disable) | ||
1694 | ftrace_hash_rec_disable(ops, 1); | ||
1695 | |||
1696 | if (ops != &global_ops || !global_start_up) | ||
1697 | ops->flags &= ~FTRACE_OPS_FL_ENABLED; | ||
1698 | |||
1212 | if (!ftrace_start_up) | 1699 | if (!ftrace_start_up) |
1213 | command |= FTRACE_DISABLE_CALLS; | 1700 | command |= FTRACE_DISABLE_CALLS; |
1214 | 1701 | ||
@@ -1273,10 +1760,10 @@ static int ftrace_update_code(struct module *mod) | |||
1273 | */ | 1760 | */ |
1274 | if (!ftrace_code_disable(mod, p)) { | 1761 | if (!ftrace_code_disable(mod, p)) { |
1275 | ftrace_free_rec(p); | 1762 | ftrace_free_rec(p); |
1276 | continue; | 1763 | /* Game over */ |
1764 | break; | ||
1277 | } | 1765 | } |
1278 | 1766 | ||
1279 | p->flags |= FTRACE_FL_CONVERTED; | ||
1280 | ftrace_update_cnt++; | 1767 | ftrace_update_cnt++; |
1281 | 1768 | ||
1282 | /* | 1769 | /* |
@@ -1351,9 +1838,9 @@ static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) | |||
1351 | enum { | 1838 | enum { |
1352 | FTRACE_ITER_FILTER = (1 << 0), | 1839 | FTRACE_ITER_FILTER = (1 << 0), |
1353 | FTRACE_ITER_NOTRACE = (1 << 1), | 1840 | FTRACE_ITER_NOTRACE = (1 << 1), |
1354 | FTRACE_ITER_FAILURES = (1 << 2), | 1841 | FTRACE_ITER_PRINTALL = (1 << 2), |
1355 | FTRACE_ITER_PRINTALL = (1 << 3), | 1842 | FTRACE_ITER_HASH = (1 << 3), |
1356 | FTRACE_ITER_HASH = (1 << 4), | 1843 | FTRACE_ITER_ENABLED = (1 << 4), |
1357 | }; | 1844 | }; |
1358 | 1845 | ||
1359 | #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ | 1846 | #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ |
@@ -1365,6 +1852,8 @@ struct ftrace_iterator { | |||
1365 | struct dyn_ftrace *func; | 1852 | struct dyn_ftrace *func; |
1366 | struct ftrace_func_probe *probe; | 1853 | struct ftrace_func_probe *probe; |
1367 | struct trace_parser parser; | 1854 | struct trace_parser parser; |
1855 | struct ftrace_hash *hash; | ||
1856 | struct ftrace_ops *ops; | ||
1368 | int hidx; | 1857 | int hidx; |
1369 | int idx; | 1858 | int idx; |
1370 | unsigned flags; | 1859 | unsigned flags; |
@@ -1461,8 +1950,12 @@ static void * | |||
1461 | t_next(struct seq_file *m, void *v, loff_t *pos) | 1950 | t_next(struct seq_file *m, void *v, loff_t *pos) |
1462 | { | 1951 | { |
1463 | struct ftrace_iterator *iter = m->private; | 1952 | struct ftrace_iterator *iter = m->private; |
1953 | struct ftrace_ops *ops = &global_ops; | ||
1464 | struct dyn_ftrace *rec = NULL; | 1954 | struct dyn_ftrace *rec = NULL; |
1465 | 1955 | ||
1956 | if (unlikely(ftrace_disabled)) | ||
1957 | return NULL; | ||
1958 | |||
1466 | if (iter->flags & FTRACE_ITER_HASH) | 1959 | if (iter->flags & FTRACE_ITER_HASH) |
1467 | return t_hash_next(m, pos); | 1960 | return t_hash_next(m, pos); |
1468 | 1961 | ||
@@ -1483,17 +1976,15 @@ t_next(struct seq_file *m, void *v, loff_t *pos) | |||
1483 | rec = &iter->pg->records[iter->idx++]; | 1976 | rec = &iter->pg->records[iter->idx++]; |
1484 | if ((rec->flags & FTRACE_FL_FREE) || | 1977 | if ((rec->flags & FTRACE_FL_FREE) || |
1485 | 1978 | ||
1486 | (!(iter->flags & FTRACE_ITER_FAILURES) && | ||
1487 | (rec->flags & FTRACE_FL_FAILED)) || | ||
1488 | |||
1489 | ((iter->flags & FTRACE_ITER_FAILURES) && | ||
1490 | !(rec->flags & FTRACE_FL_FAILED)) || | ||
1491 | |||
1492 | ((iter->flags & FTRACE_ITER_FILTER) && | 1979 | ((iter->flags & FTRACE_ITER_FILTER) && |
1493 | !(rec->flags & FTRACE_FL_FILTER)) || | 1980 | !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) || |
1494 | 1981 | ||
1495 | ((iter->flags & FTRACE_ITER_NOTRACE) && | 1982 | ((iter->flags & FTRACE_ITER_NOTRACE) && |
1496 | !(rec->flags & FTRACE_FL_NOTRACE))) { | 1983 | !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) || |
1984 | |||
1985 | ((iter->flags & FTRACE_ITER_ENABLED) && | ||
1986 | !(rec->flags & ~FTRACE_FL_MASK))) { | ||
1987 | |||
1497 | rec = NULL; | 1988 | rec = NULL; |
1498 | goto retry; | 1989 | goto retry; |
1499 | } | 1990 | } |
@@ -1517,10 +2008,15 @@ static void reset_iter_read(struct ftrace_iterator *iter) | |||
1517 | static void *t_start(struct seq_file *m, loff_t *pos) | 2008 | static void *t_start(struct seq_file *m, loff_t *pos) |
1518 | { | 2009 | { |
1519 | struct ftrace_iterator *iter = m->private; | 2010 | struct ftrace_iterator *iter = m->private; |
2011 | struct ftrace_ops *ops = &global_ops; | ||
1520 | void *p = NULL; | 2012 | void *p = NULL; |
1521 | loff_t l; | 2013 | loff_t l; |
1522 | 2014 | ||
1523 | mutex_lock(&ftrace_lock); | 2015 | mutex_lock(&ftrace_lock); |
2016 | |||
2017 | if (unlikely(ftrace_disabled)) | ||
2018 | return NULL; | ||
2019 | |||
1524 | /* | 2020 | /* |
1525 | * If an lseek was done, then reset and start from beginning. | 2021 | * If an lseek was done, then reset and start from beginning. |
1526 | */ | 2022 | */ |
@@ -1532,7 +2028,7 @@ static void *t_start(struct seq_file *m, loff_t *pos) | |||
1532 | * off, we can short cut and just print out that all | 2028 | * off, we can short cut and just print out that all |
1533 | * functions are enabled. | 2029 | * functions are enabled. |
1534 | */ | 2030 | */ |
1535 | if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) { | 2031 | if (iter->flags & FTRACE_ITER_FILTER && !ops->filter_hash->count) { |
1536 | if (*pos > 0) | 2032 | if (*pos > 0) |
1537 | return t_hash_start(m, pos); | 2033 | return t_hash_start(m, pos); |
1538 | iter->flags |= FTRACE_ITER_PRINTALL; | 2034 | iter->flags |= FTRACE_ITER_PRINTALL; |
@@ -1590,7 +2086,11 @@ static int t_show(struct seq_file *m, void *v) | |||
1590 | if (!rec) | 2086 | if (!rec) |
1591 | return 0; | 2087 | return 0; |
1592 | 2088 | ||
1593 | seq_printf(m, "%ps\n", (void *)rec->ip); | 2089 | seq_printf(m, "%ps", (void *)rec->ip); |
2090 | if (iter->flags & FTRACE_ITER_ENABLED) | ||
2091 | seq_printf(m, " (%ld)", | ||
2092 | rec->flags & ~FTRACE_FL_MASK); | ||
2093 | seq_printf(m, "\n"); | ||
1594 | 2094 | ||
1595 | return 0; | 2095 | return 0; |
1596 | } | 2096 | } |
@@ -1630,44 +2130,46 @@ ftrace_avail_open(struct inode *inode, struct file *file) | |||
1630 | } | 2130 | } |
1631 | 2131 | ||
1632 | static int | 2132 | static int |
1633 | ftrace_failures_open(struct inode *inode, struct file *file) | 2133 | ftrace_enabled_open(struct inode *inode, struct file *file) |
1634 | { | 2134 | { |
1635 | int ret; | ||
1636 | struct seq_file *m; | ||
1637 | struct ftrace_iterator *iter; | 2135 | struct ftrace_iterator *iter; |
2136 | int ret; | ||
2137 | |||
2138 | if (unlikely(ftrace_disabled)) | ||
2139 | return -ENODEV; | ||
2140 | |||
2141 | iter = kzalloc(sizeof(*iter), GFP_KERNEL); | ||
2142 | if (!iter) | ||
2143 | return -ENOMEM; | ||
2144 | |||
2145 | iter->pg = ftrace_pages_start; | ||
2146 | iter->flags = FTRACE_ITER_ENABLED; | ||
1638 | 2147 | ||
1639 | ret = ftrace_avail_open(inode, file); | 2148 | ret = seq_open(file, &show_ftrace_seq_ops); |
1640 | if (!ret) { | 2149 | if (!ret) { |
1641 | m = file->private_data; | 2150 | struct seq_file *m = file->private_data; |
1642 | iter = m->private; | 2151 | |
1643 | iter->flags = FTRACE_ITER_FAILURES; | 2152 | m->private = iter; |
2153 | } else { | ||
2154 | kfree(iter); | ||
1644 | } | 2155 | } |
1645 | 2156 | ||
1646 | return ret; | 2157 | return ret; |
1647 | } | 2158 | } |
1648 | 2159 | ||
1649 | 2160 | static void ftrace_filter_reset(struct ftrace_hash *hash) | |
1650 | static void ftrace_filter_reset(int enable) | ||
1651 | { | 2161 | { |
1652 | struct ftrace_page *pg; | ||
1653 | struct dyn_ftrace *rec; | ||
1654 | unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; | ||
1655 | |||
1656 | mutex_lock(&ftrace_lock); | 2162 | mutex_lock(&ftrace_lock); |
1657 | if (enable) | 2163 | ftrace_hash_clear(hash); |
1658 | ftrace_filtered = 0; | ||
1659 | do_for_each_ftrace_rec(pg, rec) { | ||
1660 | if (rec->flags & FTRACE_FL_FAILED) | ||
1661 | continue; | ||
1662 | rec->flags &= ~type; | ||
1663 | } while_for_each_ftrace_rec(); | ||
1664 | mutex_unlock(&ftrace_lock); | 2164 | mutex_unlock(&ftrace_lock); |
1665 | } | 2165 | } |
1666 | 2166 | ||
1667 | static int | 2167 | static int |
1668 | ftrace_regex_open(struct inode *inode, struct file *file, int enable) | 2168 | ftrace_regex_open(struct ftrace_ops *ops, int flag, |
2169 | struct inode *inode, struct file *file) | ||
1669 | { | 2170 | { |
1670 | struct ftrace_iterator *iter; | 2171 | struct ftrace_iterator *iter; |
2172 | struct ftrace_hash *hash; | ||
1671 | int ret = 0; | 2173 | int ret = 0; |
1672 | 2174 | ||
1673 | if (unlikely(ftrace_disabled)) | 2175 | if (unlikely(ftrace_disabled)) |
@@ -1682,21 +2184,42 @@ ftrace_regex_open(struct inode *inode, struct file *file, int enable) | |||
1682 | return -ENOMEM; | 2184 | return -ENOMEM; |
1683 | } | 2185 | } |
1684 | 2186 | ||
2187 | if (flag & FTRACE_ITER_NOTRACE) | ||
2188 | hash = ops->notrace_hash; | ||
2189 | else | ||
2190 | hash = ops->filter_hash; | ||
2191 | |||
2192 | iter->ops = ops; | ||
2193 | iter->flags = flag; | ||
2194 | |||
2195 | if (file->f_mode & FMODE_WRITE) { | ||
2196 | mutex_lock(&ftrace_lock); | ||
2197 | iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash); | ||
2198 | mutex_unlock(&ftrace_lock); | ||
2199 | |||
2200 | if (!iter->hash) { | ||
2201 | trace_parser_put(&iter->parser); | ||
2202 | kfree(iter); | ||
2203 | return -ENOMEM; | ||
2204 | } | ||
2205 | } | ||
2206 | |||
1685 | mutex_lock(&ftrace_regex_lock); | 2207 | mutex_lock(&ftrace_regex_lock); |
2208 | |||
1686 | if ((file->f_mode & FMODE_WRITE) && | 2209 | if ((file->f_mode & FMODE_WRITE) && |
1687 | (file->f_flags & O_TRUNC)) | 2210 | (file->f_flags & O_TRUNC)) |
1688 | ftrace_filter_reset(enable); | 2211 | ftrace_filter_reset(iter->hash); |
1689 | 2212 | ||
1690 | if (file->f_mode & FMODE_READ) { | 2213 | if (file->f_mode & FMODE_READ) { |
1691 | iter->pg = ftrace_pages_start; | 2214 | iter->pg = ftrace_pages_start; |
1692 | iter->flags = enable ? FTRACE_ITER_FILTER : | ||
1693 | FTRACE_ITER_NOTRACE; | ||
1694 | 2215 | ||
1695 | ret = seq_open(file, &show_ftrace_seq_ops); | 2216 | ret = seq_open(file, &show_ftrace_seq_ops); |
1696 | if (!ret) { | 2217 | if (!ret) { |
1697 | struct seq_file *m = file->private_data; | 2218 | struct seq_file *m = file->private_data; |
1698 | m->private = iter; | 2219 | m->private = iter; |
1699 | } else { | 2220 | } else { |
2221 | /* Failed */ | ||
2222 | free_ftrace_hash(iter->hash); | ||
1700 | trace_parser_put(&iter->parser); | 2223 | trace_parser_put(&iter->parser); |
1701 | kfree(iter); | 2224 | kfree(iter); |
1702 | } | 2225 | } |
@@ -1710,13 +2233,15 @@ ftrace_regex_open(struct inode *inode, struct file *file, int enable) | |||
1710 | static int | 2233 | static int |
1711 | ftrace_filter_open(struct inode *inode, struct file *file) | 2234 | ftrace_filter_open(struct inode *inode, struct file *file) |
1712 | { | 2235 | { |
1713 | return ftrace_regex_open(inode, file, 1); | 2236 | return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER, |
2237 | inode, file); | ||
1714 | } | 2238 | } |
1715 | 2239 | ||
1716 | static int | 2240 | static int |
1717 | ftrace_notrace_open(struct inode *inode, struct file *file) | 2241 | ftrace_notrace_open(struct inode *inode, struct file *file) |
1718 | { | 2242 | { |
1719 | return ftrace_regex_open(inode, file, 0); | 2243 | return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE, |
2244 | inode, file); | ||
1720 | } | 2245 | } |
1721 | 2246 | ||
1722 | static loff_t | 2247 | static loff_t |
@@ -1761,86 +2286,99 @@ static int ftrace_match(char *str, char *regex, int len, int type) | |||
1761 | } | 2286 | } |
1762 | 2287 | ||
1763 | static int | 2288 | static int |
1764 | ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type) | 2289 | enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not) |
2290 | { | ||
2291 | struct ftrace_func_entry *entry; | ||
2292 | int ret = 0; | ||
2293 | |||
2294 | entry = ftrace_lookup_ip(hash, rec->ip); | ||
2295 | if (not) { | ||
2296 | /* Do nothing if it doesn't exist */ | ||
2297 | if (!entry) | ||
2298 | return 0; | ||
2299 | |||
2300 | free_hash_entry(hash, entry); | ||
2301 | } else { | ||
2302 | /* Do nothing if it exists */ | ||
2303 | if (entry) | ||
2304 | return 0; | ||
2305 | |||
2306 | ret = add_hash_entry(hash, rec->ip); | ||
2307 | } | ||
2308 | return ret; | ||
2309 | } | ||
2310 | |||
2311 | static int | ||
2312 | ftrace_match_record(struct dyn_ftrace *rec, char *mod, | ||
2313 | char *regex, int len, int type) | ||
1765 | { | 2314 | { |
1766 | char str[KSYM_SYMBOL_LEN]; | 2315 | char str[KSYM_SYMBOL_LEN]; |
2316 | char *modname; | ||
2317 | |||
2318 | kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); | ||
2319 | |||
2320 | if (mod) { | ||
2321 | /* module lookup requires matching the module */ | ||
2322 | if (!modname || strcmp(modname, mod)) | ||
2323 | return 0; | ||
2324 | |||
2325 | /* blank search means to match all funcs in the mod */ | ||
2326 | if (!len) | ||
2327 | return 1; | ||
2328 | } | ||
1767 | 2329 | ||
1768 | kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); | ||
1769 | return ftrace_match(str, regex, len, type); | 2330 | return ftrace_match(str, regex, len, type); |
1770 | } | 2331 | } |
1771 | 2332 | ||
1772 | static int ftrace_match_records(char *buff, int len, int enable) | 2333 | static int |
2334 | match_records(struct ftrace_hash *hash, char *buff, | ||
2335 | int len, char *mod, int not) | ||
1773 | { | 2336 | { |
1774 | unsigned int search_len; | 2337 | unsigned search_len = 0; |
1775 | struct ftrace_page *pg; | 2338 | struct ftrace_page *pg; |
1776 | struct dyn_ftrace *rec; | 2339 | struct dyn_ftrace *rec; |
1777 | unsigned long flag; | 2340 | int type = MATCH_FULL; |
1778 | char *search; | 2341 | char *search = buff; |
1779 | int type; | ||
1780 | int not; | ||
1781 | int found = 0; | 2342 | int found = 0; |
2343 | int ret; | ||
1782 | 2344 | ||
1783 | flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; | 2345 | if (len) { |
1784 | type = filter_parse_regex(buff, len, &search, ¬); | 2346 | type = filter_parse_regex(buff, len, &search, ¬); |
1785 | 2347 | search_len = strlen(search); | |
1786 | search_len = strlen(search); | 2348 | } |
1787 | 2349 | ||
1788 | mutex_lock(&ftrace_lock); | 2350 | mutex_lock(&ftrace_lock); |
1789 | do_for_each_ftrace_rec(pg, rec) { | ||
1790 | 2351 | ||
1791 | if (rec->flags & FTRACE_FL_FAILED) | 2352 | if (unlikely(ftrace_disabled)) |
1792 | continue; | 2353 | goto out_unlock; |
1793 | 2354 | ||
1794 | if (ftrace_match_record(rec, search, search_len, type)) { | 2355 | do_for_each_ftrace_rec(pg, rec) { |
1795 | if (not) | 2356 | |
1796 | rec->flags &= ~flag; | 2357 | if (ftrace_match_record(rec, mod, search, search_len, type)) { |
1797 | else | 2358 | ret = enter_record(hash, rec, not); |
1798 | rec->flags |= flag; | 2359 | if (ret < 0) { |
2360 | found = ret; | ||
2361 | goto out_unlock; | ||
2362 | } | ||
1799 | found = 1; | 2363 | found = 1; |
1800 | } | 2364 | } |
1801 | /* | ||
1802 | * Only enable filtering if we have a function that | ||
1803 | * is filtered on. | ||
1804 | */ | ||
1805 | if (enable && (rec->flags & FTRACE_FL_FILTER)) | ||
1806 | ftrace_filtered = 1; | ||
1807 | } while_for_each_ftrace_rec(); | 2365 | } while_for_each_ftrace_rec(); |
2366 | out_unlock: | ||
1808 | mutex_unlock(&ftrace_lock); | 2367 | mutex_unlock(&ftrace_lock); |
1809 | 2368 | ||
1810 | return found; | 2369 | return found; |
1811 | } | 2370 | } |
1812 | 2371 | ||
1813 | static int | 2372 | static int |
1814 | ftrace_match_module_record(struct dyn_ftrace *rec, char *mod, | 2373 | ftrace_match_records(struct ftrace_hash *hash, char *buff, int len) |
1815 | char *regex, int len, int type) | ||
1816 | { | 2374 | { |
1817 | char str[KSYM_SYMBOL_LEN]; | 2375 | return match_records(hash, buff, len, NULL, 0); |
1818 | char *modname; | ||
1819 | |||
1820 | kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); | ||
1821 | |||
1822 | if (!modname || strcmp(modname, mod)) | ||
1823 | return 0; | ||
1824 | |||
1825 | /* blank search means to match all funcs in the mod */ | ||
1826 | if (len) | ||
1827 | return ftrace_match(str, regex, len, type); | ||
1828 | else | ||
1829 | return 1; | ||
1830 | } | 2376 | } |
1831 | 2377 | ||
1832 | static int ftrace_match_module_records(char *buff, char *mod, int enable) | 2378 | static int |
2379 | ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod) | ||
1833 | { | 2380 | { |
1834 | unsigned search_len = 0; | ||
1835 | struct ftrace_page *pg; | ||
1836 | struct dyn_ftrace *rec; | ||
1837 | int type = MATCH_FULL; | ||
1838 | char *search = buff; | ||
1839 | unsigned long flag; | ||
1840 | int not = 0; | 2381 | int not = 0; |
1841 | int found = 0; | ||
1842 | |||
1843 | flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; | ||
1844 | 2382 | ||
1845 | /* blank or '*' mean the same */ | 2383 | /* blank or '*' mean the same */ |
1846 | if (strcmp(buff, "*") == 0) | 2384 | if (strcmp(buff, "*") == 0) |
@@ -1852,32 +2390,7 @@ static int ftrace_match_module_records(char *buff, char *mod, int enable) | |||
1852 | not = 1; | 2390 | not = 1; |
1853 | } | 2391 | } |
1854 | 2392 | ||
1855 | if (strlen(buff)) { | 2393 | return match_records(hash, buff, strlen(buff), mod, not); |
1856 | type = filter_parse_regex(buff, strlen(buff), &search, ¬); | ||
1857 | search_len = strlen(search); | ||
1858 | } | ||
1859 | |||
1860 | mutex_lock(&ftrace_lock); | ||
1861 | do_for_each_ftrace_rec(pg, rec) { | ||
1862 | |||
1863 | if (rec->flags & FTRACE_FL_FAILED) | ||
1864 | continue; | ||
1865 | |||
1866 | if (ftrace_match_module_record(rec, mod, | ||
1867 | search, search_len, type)) { | ||
1868 | if (not) | ||
1869 | rec->flags &= ~flag; | ||
1870 | else | ||
1871 | rec->flags |= flag; | ||
1872 | found = 1; | ||
1873 | } | ||
1874 | if (enable && (rec->flags & FTRACE_FL_FILTER)) | ||
1875 | ftrace_filtered = 1; | ||
1876 | |||
1877 | } while_for_each_ftrace_rec(); | ||
1878 | mutex_unlock(&ftrace_lock); | ||
1879 | |||
1880 | return found; | ||
1881 | } | 2394 | } |
1882 | 2395 | ||
1883 | /* | 2396 | /* |
@@ -1888,7 +2401,10 @@ static int ftrace_match_module_records(char *buff, char *mod, int enable) | |||
1888 | static int | 2401 | static int |
1889 | ftrace_mod_callback(char *func, char *cmd, char *param, int enable) | 2402 | ftrace_mod_callback(char *func, char *cmd, char *param, int enable) |
1890 | { | 2403 | { |
2404 | struct ftrace_ops *ops = &global_ops; | ||
2405 | struct ftrace_hash *hash; | ||
1891 | char *mod; | 2406 | char *mod; |
2407 | int ret = -EINVAL; | ||
1892 | 2408 | ||
1893 | /* | 2409 | /* |
1894 | * cmd == 'mod' because we only registered this func | 2410 | * cmd == 'mod' because we only registered this func |
@@ -1900,15 +2416,24 @@ ftrace_mod_callback(char *func, char *cmd, char *param, int enable) | |||
1900 | 2416 | ||
1901 | /* we must have a module name */ | 2417 | /* we must have a module name */ |
1902 | if (!param) | 2418 | if (!param) |
1903 | return -EINVAL; | 2419 | return ret; |
1904 | 2420 | ||
1905 | mod = strsep(¶m, ":"); | 2421 | mod = strsep(¶m, ":"); |
1906 | if (!strlen(mod)) | 2422 | if (!strlen(mod)) |
1907 | return -EINVAL; | 2423 | return ret; |
1908 | 2424 | ||
1909 | if (ftrace_match_module_records(func, mod, enable)) | 2425 | if (enable) |
1910 | return 0; | 2426 | hash = ops->filter_hash; |
1911 | return -EINVAL; | 2427 | else |
2428 | hash = ops->notrace_hash; | ||
2429 | |||
2430 | ret = ftrace_match_module_records(hash, func, mod); | ||
2431 | if (!ret) | ||
2432 | ret = -EINVAL; | ||
2433 | if (ret < 0) | ||
2434 | return ret; | ||
2435 | |||
2436 | return 0; | ||
1912 | } | 2437 | } |
1913 | 2438 | ||
1914 | static struct ftrace_func_command ftrace_mod_cmd = { | 2439 | static struct ftrace_func_command ftrace_mod_cmd = { |
@@ -1959,6 +2484,7 @@ static int ftrace_probe_registered; | |||
1959 | 2484 | ||
1960 | static void __enable_ftrace_function_probe(void) | 2485 | static void __enable_ftrace_function_probe(void) |
1961 | { | 2486 | { |
2487 | int ret; | ||
1962 | int i; | 2488 | int i; |
1963 | 2489 | ||
1964 | if (ftrace_probe_registered) | 2490 | if (ftrace_probe_registered) |
@@ -1973,13 +2499,16 @@ static void __enable_ftrace_function_probe(void) | |||
1973 | if (i == FTRACE_FUNC_HASHSIZE) | 2499 | if (i == FTRACE_FUNC_HASHSIZE) |
1974 | return; | 2500 | return; |
1975 | 2501 | ||
1976 | __register_ftrace_function(&trace_probe_ops); | 2502 | ret = __register_ftrace_function(&trace_probe_ops); |
1977 | ftrace_startup(0); | 2503 | if (!ret) |
2504 | ftrace_startup(&trace_probe_ops, 0); | ||
2505 | |||
1978 | ftrace_probe_registered = 1; | 2506 | ftrace_probe_registered = 1; |
1979 | } | 2507 | } |
1980 | 2508 | ||
1981 | static void __disable_ftrace_function_probe(void) | 2509 | static void __disable_ftrace_function_probe(void) |
1982 | { | 2510 | { |
2511 | int ret; | ||
1983 | int i; | 2512 | int i; |
1984 | 2513 | ||
1985 | if (!ftrace_probe_registered) | 2514 | if (!ftrace_probe_registered) |
@@ -1992,8 +2521,10 @@ static void __disable_ftrace_function_probe(void) | |||
1992 | } | 2521 | } |
1993 | 2522 | ||
1994 | /* no more funcs left */ | 2523 | /* no more funcs left */ |
1995 | __unregister_ftrace_function(&trace_probe_ops); | 2524 | ret = __unregister_ftrace_function(&trace_probe_ops); |
1996 | ftrace_shutdown(0); | 2525 | if (!ret) |
2526 | ftrace_shutdown(&trace_probe_ops, 0); | ||
2527 | |||
1997 | ftrace_probe_registered = 0; | 2528 | ftrace_probe_registered = 0; |
1998 | } | 2529 | } |
1999 | 2530 | ||
@@ -2029,12 +2560,13 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, | |||
2029 | return -EINVAL; | 2560 | return -EINVAL; |
2030 | 2561 | ||
2031 | mutex_lock(&ftrace_lock); | 2562 | mutex_lock(&ftrace_lock); |
2032 | do_for_each_ftrace_rec(pg, rec) { | ||
2033 | 2563 | ||
2034 | if (rec->flags & FTRACE_FL_FAILED) | 2564 | if (unlikely(ftrace_disabled)) |
2035 | continue; | 2565 | goto out_unlock; |
2566 | |||
2567 | do_for_each_ftrace_rec(pg, rec) { | ||
2036 | 2568 | ||
2037 | if (!ftrace_match_record(rec, search, len, type)) | 2569 | if (!ftrace_match_record(rec, NULL, search, len, type)) |
2038 | continue; | 2570 | continue; |
2039 | 2571 | ||
2040 | entry = kmalloc(sizeof(*entry), GFP_KERNEL); | 2572 | entry = kmalloc(sizeof(*entry), GFP_KERNEL); |
@@ -2195,18 +2727,22 @@ int unregister_ftrace_command(struct ftrace_func_command *cmd) | |||
2195 | return ret; | 2727 | return ret; |
2196 | } | 2728 | } |
2197 | 2729 | ||
2198 | static int ftrace_process_regex(char *buff, int len, int enable) | 2730 | static int ftrace_process_regex(struct ftrace_hash *hash, |
2731 | char *buff, int len, int enable) | ||
2199 | { | 2732 | { |
2200 | char *func, *command, *next = buff; | 2733 | char *func, *command, *next = buff; |
2201 | struct ftrace_func_command *p; | 2734 | struct ftrace_func_command *p; |
2202 | int ret = -EINVAL; | 2735 | int ret; |
2203 | 2736 | ||
2204 | func = strsep(&next, ":"); | 2737 | func = strsep(&next, ":"); |
2205 | 2738 | ||
2206 | if (!next) { | 2739 | if (!next) { |
2207 | if (ftrace_match_records(func, len, enable)) | 2740 | ret = ftrace_match_records(hash, func, len); |
2208 | return 0; | 2741 | if (!ret) |
2209 | return ret; | 2742 | ret = -EINVAL; |
2743 | if (ret < 0) | ||
2744 | return ret; | ||
2745 | return 0; | ||
2210 | } | 2746 | } |
2211 | 2747 | ||
2212 | /* command found */ | 2748 | /* command found */ |
@@ -2239,6 +2775,10 @@ ftrace_regex_write(struct file *file, const char __user *ubuf, | |||
2239 | 2775 | ||
2240 | mutex_lock(&ftrace_regex_lock); | 2776 | mutex_lock(&ftrace_regex_lock); |
2241 | 2777 | ||
2778 | ret = -ENODEV; | ||
2779 | if (unlikely(ftrace_disabled)) | ||
2780 | goto out_unlock; | ||
2781 | |||
2242 | if (file->f_mode & FMODE_READ) { | 2782 | if (file->f_mode & FMODE_READ) { |
2243 | struct seq_file *m = file->private_data; | 2783 | struct seq_file *m = file->private_data; |
2244 | iter = m->private; | 2784 | iter = m->private; |
@@ -2250,7 +2790,7 @@ ftrace_regex_write(struct file *file, const char __user *ubuf, | |||
2250 | 2790 | ||
2251 | if (read >= 0 && trace_parser_loaded(parser) && | 2791 | if (read >= 0 && trace_parser_loaded(parser) && |
2252 | !trace_parser_cont(parser)) { | 2792 | !trace_parser_cont(parser)) { |
2253 | ret = ftrace_process_regex(parser->buffer, | 2793 | ret = ftrace_process_regex(iter->hash, parser->buffer, |
2254 | parser->idx, enable); | 2794 | parser->idx, enable); |
2255 | trace_parser_clear(parser); | 2795 | trace_parser_clear(parser); |
2256 | if (ret) | 2796 | if (ret) |
@@ -2278,22 +2818,49 @@ ftrace_notrace_write(struct file *file, const char __user *ubuf, | |||
2278 | return ftrace_regex_write(file, ubuf, cnt, ppos, 0); | 2818 | return ftrace_regex_write(file, ubuf, cnt, ppos, 0); |
2279 | } | 2819 | } |
2280 | 2820 | ||
2281 | static void | 2821 | static int |
2282 | ftrace_set_regex(unsigned char *buf, int len, int reset, int enable) | 2822 | ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, |
2823 | int reset, int enable) | ||
2283 | { | 2824 | { |
2825 | struct ftrace_hash **orig_hash; | ||
2826 | struct ftrace_hash *hash; | ||
2827 | int ret; | ||
2828 | |||
2829 | /* All global ops uses the global ops filters */ | ||
2830 | if (ops->flags & FTRACE_OPS_FL_GLOBAL) | ||
2831 | ops = &global_ops; | ||
2832 | |||
2284 | if (unlikely(ftrace_disabled)) | 2833 | if (unlikely(ftrace_disabled)) |
2285 | return; | 2834 | return -ENODEV; |
2835 | |||
2836 | if (enable) | ||
2837 | orig_hash = &ops->filter_hash; | ||
2838 | else | ||
2839 | orig_hash = &ops->notrace_hash; | ||
2840 | |||
2841 | hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); | ||
2842 | if (!hash) | ||
2843 | return -ENOMEM; | ||
2286 | 2844 | ||
2287 | mutex_lock(&ftrace_regex_lock); | 2845 | mutex_lock(&ftrace_regex_lock); |
2288 | if (reset) | 2846 | if (reset) |
2289 | ftrace_filter_reset(enable); | 2847 | ftrace_filter_reset(hash); |
2290 | if (buf) | 2848 | if (buf) |
2291 | ftrace_match_records(buf, len, enable); | 2849 | ftrace_match_records(hash, buf, len); |
2850 | |||
2851 | mutex_lock(&ftrace_lock); | ||
2852 | ret = ftrace_hash_move(orig_hash, hash); | ||
2853 | mutex_unlock(&ftrace_lock); | ||
2854 | |||
2292 | mutex_unlock(&ftrace_regex_lock); | 2855 | mutex_unlock(&ftrace_regex_lock); |
2856 | |||
2857 | free_ftrace_hash(hash); | ||
2858 | return ret; | ||
2293 | } | 2859 | } |
2294 | 2860 | ||
2295 | /** | 2861 | /** |
2296 | * ftrace_set_filter - set a function to filter on in ftrace | 2862 | * ftrace_set_filter - set a function to filter on in ftrace |
2863 | * @ops - the ops to set the filter with | ||
2297 | * @buf - the string that holds the function filter text. | 2864 | * @buf - the string that holds the function filter text. |
2298 | * @len - the length of the string. | 2865 | * @len - the length of the string. |
2299 | * @reset - non zero to reset all filters before applying this filter. | 2866 | * @reset - non zero to reset all filters before applying this filter. |
@@ -2301,13 +2868,16 @@ ftrace_set_regex(unsigned char *buf, int len, int reset, int enable) | |||
2301 | * Filters denote which functions should be enabled when tracing is enabled. | 2868 | * Filters denote which functions should be enabled when tracing is enabled. |
2302 | * If @buf is NULL and reset is set, all functions will be enabled for tracing. | 2869 | * If @buf is NULL and reset is set, all functions will be enabled for tracing. |
2303 | */ | 2870 | */ |
2304 | void ftrace_set_filter(unsigned char *buf, int len, int reset) | 2871 | void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, |
2872 | int len, int reset) | ||
2305 | { | 2873 | { |
2306 | ftrace_set_regex(buf, len, reset, 1); | 2874 | ftrace_set_regex(ops, buf, len, reset, 1); |
2307 | } | 2875 | } |
2876 | EXPORT_SYMBOL_GPL(ftrace_set_filter); | ||
2308 | 2877 | ||
2309 | /** | 2878 | /** |
2310 | * ftrace_set_notrace - set a function to not trace in ftrace | 2879 | * ftrace_set_notrace - set a function to not trace in ftrace |
2880 | * @ops - the ops to set the notrace filter with | ||
2311 | * @buf - the string that holds the function notrace text. | 2881 | * @buf - the string that holds the function notrace text. |
2312 | * @len - the length of the string. | 2882 | * @len - the length of the string. |
2313 | * @reset - non zero to reset all filters before applying this filter. | 2883 | * @reset - non zero to reset all filters before applying this filter. |
@@ -2316,10 +2886,44 @@ void ftrace_set_filter(unsigned char *buf, int len, int reset) | |||
2316 | * is enabled. If @buf is NULL and reset is set, all functions will be enabled | 2886 | * is enabled. If @buf is NULL and reset is set, all functions will be enabled |
2317 | * for tracing. | 2887 | * for tracing. |
2318 | */ | 2888 | */ |
2319 | void ftrace_set_notrace(unsigned char *buf, int len, int reset) | 2889 | void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, |
2890 | int len, int reset) | ||
2320 | { | 2891 | { |
2321 | ftrace_set_regex(buf, len, reset, 0); | 2892 | ftrace_set_regex(ops, buf, len, reset, 0); |
2322 | } | 2893 | } |
2894 | EXPORT_SYMBOL_GPL(ftrace_set_notrace); | ||
2895 | /** | ||
2896 | * ftrace_set_filter - set a function to filter on in ftrace | ||
2897 | * @ops - the ops to set the filter with | ||
2898 | * @buf - the string that holds the function filter text. | ||
2899 | * @len - the length of the string. | ||
2900 | * @reset - non zero to reset all filters before applying this filter. | ||
2901 | * | ||
2902 | * Filters denote which functions should be enabled when tracing is enabled. | ||
2903 | * If @buf is NULL and reset is set, all functions will be enabled for tracing. | ||
2904 | */ | ||
2905 | void ftrace_set_global_filter(unsigned char *buf, int len, int reset) | ||
2906 | { | ||
2907 | ftrace_set_regex(&global_ops, buf, len, reset, 1); | ||
2908 | } | ||
2909 | EXPORT_SYMBOL_GPL(ftrace_set_global_filter); | ||
2910 | |||
2911 | /** | ||
2912 | * ftrace_set_notrace - set a function to not trace in ftrace | ||
2913 | * @ops - the ops to set the notrace filter with | ||
2914 | * @buf - the string that holds the function notrace text. | ||
2915 | * @len - the length of the string. | ||
2916 | * @reset - non zero to reset all filters before applying this filter. | ||
2917 | * | ||
2918 | * Notrace Filters denote which functions should not be enabled when tracing | ||
2919 | * is enabled. If @buf is NULL and reset is set, all functions will be enabled | ||
2920 | * for tracing. | ||
2921 | */ | ||
2922 | void ftrace_set_global_notrace(unsigned char *buf, int len, int reset) | ||
2923 | { | ||
2924 | ftrace_set_regex(&global_ops, buf, len, reset, 0); | ||
2925 | } | ||
2926 | EXPORT_SYMBOL_GPL(ftrace_set_global_notrace); | ||
2323 | 2927 | ||
2324 | /* | 2928 | /* |
2325 | * command line interface to allow users to set filters on boot up. | 2929 | * command line interface to allow users to set filters on boot up. |
@@ -2370,22 +2974,23 @@ static void __init set_ftrace_early_graph(char *buf) | |||
2370 | } | 2974 | } |
2371 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | 2975 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
2372 | 2976 | ||
2373 | static void __init set_ftrace_early_filter(char *buf, int enable) | 2977 | static void __init |
2978 | set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable) | ||
2374 | { | 2979 | { |
2375 | char *func; | 2980 | char *func; |
2376 | 2981 | ||
2377 | while (buf) { | 2982 | while (buf) { |
2378 | func = strsep(&buf, ","); | 2983 | func = strsep(&buf, ","); |
2379 | ftrace_set_regex(func, strlen(func), 0, enable); | 2984 | ftrace_set_regex(ops, func, strlen(func), 0, enable); |
2380 | } | 2985 | } |
2381 | } | 2986 | } |
2382 | 2987 | ||
2383 | static void __init set_ftrace_early_filters(void) | 2988 | static void __init set_ftrace_early_filters(void) |
2384 | { | 2989 | { |
2385 | if (ftrace_filter_buf[0]) | 2990 | if (ftrace_filter_buf[0]) |
2386 | set_ftrace_early_filter(ftrace_filter_buf, 1); | 2991 | set_ftrace_early_filter(&global_ops, ftrace_filter_buf, 1); |
2387 | if (ftrace_notrace_buf[0]) | 2992 | if (ftrace_notrace_buf[0]) |
2388 | set_ftrace_early_filter(ftrace_notrace_buf, 0); | 2993 | set_ftrace_early_filter(&global_ops, ftrace_notrace_buf, 0); |
2389 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 2994 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
2390 | if (ftrace_graph_buf[0]) | 2995 | if (ftrace_graph_buf[0]) |
2391 | set_ftrace_early_graph(ftrace_graph_buf); | 2996 | set_ftrace_early_graph(ftrace_graph_buf); |
@@ -2393,11 +2998,14 @@ static void __init set_ftrace_early_filters(void) | |||
2393 | } | 2998 | } |
2394 | 2999 | ||
2395 | static int | 3000 | static int |
2396 | ftrace_regex_release(struct inode *inode, struct file *file, int enable) | 3001 | ftrace_regex_release(struct inode *inode, struct file *file) |
2397 | { | 3002 | { |
2398 | struct seq_file *m = (struct seq_file *)file->private_data; | 3003 | struct seq_file *m = (struct seq_file *)file->private_data; |
2399 | struct ftrace_iterator *iter; | 3004 | struct ftrace_iterator *iter; |
3005 | struct ftrace_hash **orig_hash; | ||
2400 | struct trace_parser *parser; | 3006 | struct trace_parser *parser; |
3007 | int filter_hash; | ||
3008 | int ret; | ||
2401 | 3009 | ||
2402 | mutex_lock(&ftrace_regex_lock); | 3010 | mutex_lock(&ftrace_regex_lock); |
2403 | if (file->f_mode & FMODE_READ) { | 3011 | if (file->f_mode & FMODE_READ) { |
@@ -2410,33 +3018,41 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable) | |||
2410 | parser = &iter->parser; | 3018 | parser = &iter->parser; |
2411 | if (trace_parser_loaded(parser)) { | 3019 | if (trace_parser_loaded(parser)) { |
2412 | parser->buffer[parser->idx] = 0; | 3020 | parser->buffer[parser->idx] = 0; |
2413 | ftrace_match_records(parser->buffer, parser->idx, enable); | 3021 | ftrace_match_records(iter->hash, parser->buffer, parser->idx); |
2414 | } | 3022 | } |
2415 | 3023 | ||
2416 | mutex_lock(&ftrace_lock); | ||
2417 | if (ftrace_start_up && ftrace_enabled) | ||
2418 | ftrace_run_update_code(FTRACE_ENABLE_CALLS); | ||
2419 | mutex_unlock(&ftrace_lock); | ||
2420 | |||
2421 | trace_parser_put(parser); | 3024 | trace_parser_put(parser); |
3025 | |||
3026 | if (file->f_mode & FMODE_WRITE) { | ||
3027 | filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); | ||
3028 | |||
3029 | if (filter_hash) | ||
3030 | orig_hash = &iter->ops->filter_hash; | ||
3031 | else | ||
3032 | orig_hash = &iter->ops->notrace_hash; | ||
3033 | |||
3034 | mutex_lock(&ftrace_lock); | ||
3035 | /* | ||
3036 | * Remove the current set, update the hash and add | ||
3037 | * them back. | ||
3038 | */ | ||
3039 | ftrace_hash_rec_disable(iter->ops, filter_hash); | ||
3040 | ret = ftrace_hash_move(orig_hash, iter->hash); | ||
3041 | if (!ret) { | ||
3042 | ftrace_hash_rec_enable(iter->ops, filter_hash); | ||
3043 | if (iter->ops->flags & FTRACE_OPS_FL_ENABLED | ||
3044 | && ftrace_enabled) | ||
3045 | ftrace_run_update_code(FTRACE_ENABLE_CALLS); | ||
3046 | } | ||
3047 | mutex_unlock(&ftrace_lock); | ||
3048 | } | ||
3049 | free_ftrace_hash(iter->hash); | ||
2422 | kfree(iter); | 3050 | kfree(iter); |
2423 | 3051 | ||
2424 | mutex_unlock(&ftrace_regex_lock); | 3052 | mutex_unlock(&ftrace_regex_lock); |
2425 | return 0; | 3053 | return 0; |
2426 | } | 3054 | } |
2427 | 3055 | ||
2428 | static int | ||
2429 | ftrace_filter_release(struct inode *inode, struct file *file) | ||
2430 | { | ||
2431 | return ftrace_regex_release(inode, file, 1); | ||
2432 | } | ||
2433 | |||
2434 | static int | ||
2435 | ftrace_notrace_release(struct inode *inode, struct file *file) | ||
2436 | { | ||
2437 | return ftrace_regex_release(inode, file, 0); | ||
2438 | } | ||
2439 | |||
2440 | static const struct file_operations ftrace_avail_fops = { | 3056 | static const struct file_operations ftrace_avail_fops = { |
2441 | .open = ftrace_avail_open, | 3057 | .open = ftrace_avail_open, |
2442 | .read = seq_read, | 3058 | .read = seq_read, |
@@ -2444,8 +3060,8 @@ static const struct file_operations ftrace_avail_fops = { | |||
2444 | .release = seq_release_private, | 3060 | .release = seq_release_private, |
2445 | }; | 3061 | }; |
2446 | 3062 | ||
2447 | static const struct file_operations ftrace_failures_fops = { | 3063 | static const struct file_operations ftrace_enabled_fops = { |
2448 | .open = ftrace_failures_open, | 3064 | .open = ftrace_enabled_open, |
2449 | .read = seq_read, | 3065 | .read = seq_read, |
2450 | .llseek = seq_lseek, | 3066 | .llseek = seq_lseek, |
2451 | .release = seq_release_private, | 3067 | .release = seq_release_private, |
@@ -2456,7 +3072,7 @@ static const struct file_operations ftrace_filter_fops = { | |||
2456 | .read = seq_read, | 3072 | .read = seq_read, |
2457 | .write = ftrace_filter_write, | 3073 | .write = ftrace_filter_write, |
2458 | .llseek = ftrace_regex_lseek, | 3074 | .llseek = ftrace_regex_lseek, |
2459 | .release = ftrace_filter_release, | 3075 | .release = ftrace_regex_release, |
2460 | }; | 3076 | }; |
2461 | 3077 | ||
2462 | static const struct file_operations ftrace_notrace_fops = { | 3078 | static const struct file_operations ftrace_notrace_fops = { |
@@ -2464,7 +3080,7 @@ static const struct file_operations ftrace_notrace_fops = { | |||
2464 | .read = seq_read, | 3080 | .read = seq_read, |
2465 | .write = ftrace_notrace_write, | 3081 | .write = ftrace_notrace_write, |
2466 | .llseek = ftrace_regex_lseek, | 3082 | .llseek = ftrace_regex_lseek, |
2467 | .release = ftrace_notrace_release, | 3083 | .release = ftrace_regex_release, |
2468 | }; | 3084 | }; |
2469 | 3085 | ||
2470 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 3086 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
@@ -2573,9 +3189,6 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer) | |||
2573 | bool exists; | 3189 | bool exists; |
2574 | int i; | 3190 | int i; |
2575 | 3191 | ||
2576 | if (ftrace_disabled) | ||
2577 | return -ENODEV; | ||
2578 | |||
2579 | /* decode regex */ | 3192 | /* decode regex */ |
2580 | type = filter_parse_regex(buffer, strlen(buffer), &search, ¬); | 3193 | type = filter_parse_regex(buffer, strlen(buffer), &search, ¬); |
2581 | if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS) | 3194 | if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS) |
@@ -2584,12 +3197,18 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer) | |||
2584 | search_len = strlen(search); | 3197 | search_len = strlen(search); |
2585 | 3198 | ||
2586 | mutex_lock(&ftrace_lock); | 3199 | mutex_lock(&ftrace_lock); |
3200 | |||
3201 | if (unlikely(ftrace_disabled)) { | ||
3202 | mutex_unlock(&ftrace_lock); | ||
3203 | return -ENODEV; | ||
3204 | } | ||
3205 | |||
2587 | do_for_each_ftrace_rec(pg, rec) { | 3206 | do_for_each_ftrace_rec(pg, rec) { |
2588 | 3207 | ||
2589 | if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) | 3208 | if (rec->flags & FTRACE_FL_FREE) |
2590 | continue; | 3209 | continue; |
2591 | 3210 | ||
2592 | if (ftrace_match_record(rec, search, search_len, type)) { | 3211 | if (ftrace_match_record(rec, NULL, search, search_len, type)) { |
2593 | /* if it is in the array */ | 3212 | /* if it is in the array */ |
2594 | exists = false; | 3213 | exists = false; |
2595 | for (i = 0; i < *idx; i++) { | 3214 | for (i = 0; i < *idx; i++) { |
@@ -2679,8 +3298,8 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer) | |||
2679 | trace_create_file("available_filter_functions", 0444, | 3298 | trace_create_file("available_filter_functions", 0444, |
2680 | d_tracer, NULL, &ftrace_avail_fops); | 3299 | d_tracer, NULL, &ftrace_avail_fops); |
2681 | 3300 | ||
2682 | trace_create_file("failures", 0444, | 3301 | trace_create_file("enabled_functions", 0444, |
2683 | d_tracer, NULL, &ftrace_failures_fops); | 3302 | d_tracer, NULL, &ftrace_enabled_fops); |
2684 | 3303 | ||
2685 | trace_create_file("set_ftrace_filter", 0644, d_tracer, | 3304 | trace_create_file("set_ftrace_filter", 0644, d_tracer, |
2686 | NULL, &ftrace_filter_fops); | 3305 | NULL, &ftrace_filter_fops); |
@@ -2703,7 +3322,6 @@ static int ftrace_process_locs(struct module *mod, | |||
2703 | { | 3322 | { |
2704 | unsigned long *p; | 3323 | unsigned long *p; |
2705 | unsigned long addr; | 3324 | unsigned long addr; |
2706 | unsigned long flags; | ||
2707 | 3325 | ||
2708 | mutex_lock(&ftrace_lock); | 3326 | mutex_lock(&ftrace_lock); |
2709 | p = start; | 3327 | p = start; |
@@ -2720,10 +3338,7 @@ static int ftrace_process_locs(struct module *mod, | |||
2720 | ftrace_record_ip(addr); | 3338 | ftrace_record_ip(addr); |
2721 | } | 3339 | } |
2722 | 3340 | ||
2723 | /* disable interrupts to prevent kstop machine */ | ||
2724 | local_irq_save(flags); | ||
2725 | ftrace_update_code(mod); | 3341 | ftrace_update_code(mod); |
2726 | local_irq_restore(flags); | ||
2727 | mutex_unlock(&ftrace_lock); | 3342 | mutex_unlock(&ftrace_lock); |
2728 | 3343 | ||
2729 | return 0; | 3344 | return 0; |
@@ -2735,10 +3350,11 @@ void ftrace_release_mod(struct module *mod) | |||
2735 | struct dyn_ftrace *rec; | 3350 | struct dyn_ftrace *rec; |
2736 | struct ftrace_page *pg; | 3351 | struct ftrace_page *pg; |
2737 | 3352 | ||
3353 | mutex_lock(&ftrace_lock); | ||
3354 | |||
2738 | if (ftrace_disabled) | 3355 | if (ftrace_disabled) |
2739 | return; | 3356 | goto out_unlock; |
2740 | 3357 | ||
2741 | mutex_lock(&ftrace_lock); | ||
2742 | do_for_each_ftrace_rec(pg, rec) { | 3358 | do_for_each_ftrace_rec(pg, rec) { |
2743 | if (within_module_core(rec->ip, mod)) { | 3359 | if (within_module_core(rec->ip, mod)) { |
2744 | /* | 3360 | /* |
@@ -2749,6 +3365,7 @@ void ftrace_release_mod(struct module *mod) | |||
2749 | ftrace_free_rec(rec); | 3365 | ftrace_free_rec(rec); |
2750 | } | 3366 | } |
2751 | } while_for_each_ftrace_rec(); | 3367 | } while_for_each_ftrace_rec(); |
3368 | out_unlock: | ||
2752 | mutex_unlock(&ftrace_lock); | 3369 | mutex_unlock(&ftrace_lock); |
2753 | } | 3370 | } |
2754 | 3371 | ||
@@ -2835,6 +3452,10 @@ void __init ftrace_init(void) | |||
2835 | 3452 | ||
2836 | #else | 3453 | #else |
2837 | 3454 | ||
3455 | static struct ftrace_ops global_ops = { | ||
3456 | .func = ftrace_stub, | ||
3457 | }; | ||
3458 | |||
2838 | static int __init ftrace_nodyn_init(void) | 3459 | static int __init ftrace_nodyn_init(void) |
2839 | { | 3460 | { |
2840 | ftrace_enabled = 1; | 3461 | ftrace_enabled = 1; |
@@ -2845,12 +3466,38 @@ device_initcall(ftrace_nodyn_init); | |||
2845 | static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; } | 3466 | static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; } |
2846 | static inline void ftrace_startup_enable(int command) { } | 3467 | static inline void ftrace_startup_enable(int command) { } |
2847 | /* Keep as macros so we do not need to define the commands */ | 3468 | /* Keep as macros so we do not need to define the commands */ |
2848 | # define ftrace_startup(command) do { } while (0) | 3469 | # define ftrace_startup(ops, command) do { } while (0) |
2849 | # define ftrace_shutdown(command) do { } while (0) | 3470 | # define ftrace_shutdown(ops, command) do { } while (0) |
2850 | # define ftrace_startup_sysctl() do { } while (0) | 3471 | # define ftrace_startup_sysctl() do { } while (0) |
2851 | # define ftrace_shutdown_sysctl() do { } while (0) | 3472 | # define ftrace_shutdown_sysctl() do { } while (0) |
3473 | |||
3474 | static inline int | ||
3475 | ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip) | ||
3476 | { | ||
3477 | return 1; | ||
3478 | } | ||
3479 | |||
2852 | #endif /* CONFIG_DYNAMIC_FTRACE */ | 3480 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
2853 | 3481 | ||
3482 | static void | ||
3483 | ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip) | ||
3484 | { | ||
3485 | struct ftrace_ops *op; | ||
3486 | |||
3487 | /* | ||
3488 | * Some of the ops may be dynamically allocated, | ||
3489 | * they must be freed after a synchronize_sched(). | ||
3490 | */ | ||
3491 | preempt_disable_notrace(); | ||
3492 | op = rcu_dereference_raw(ftrace_ops_list); | ||
3493 | while (op != &ftrace_list_end) { | ||
3494 | if (ftrace_ops_test(op, ip)) | ||
3495 | op->func(ip, parent_ip); | ||
3496 | op = rcu_dereference_raw(op->next); | ||
3497 | }; | ||
3498 | preempt_enable_notrace(); | ||
3499 | } | ||
3500 | |||
2854 | static void clear_ftrace_swapper(void) | 3501 | static void clear_ftrace_swapper(void) |
2855 | { | 3502 | { |
2856 | struct task_struct *p; | 3503 | struct task_struct *p; |
@@ -3143,19 +3790,23 @@ void ftrace_kill(void) | |||
3143 | */ | 3790 | */ |
3144 | int register_ftrace_function(struct ftrace_ops *ops) | 3791 | int register_ftrace_function(struct ftrace_ops *ops) |
3145 | { | 3792 | { |
3146 | int ret; | 3793 | int ret = -1; |
3147 | |||
3148 | if (unlikely(ftrace_disabled)) | ||
3149 | return -1; | ||
3150 | 3794 | ||
3151 | mutex_lock(&ftrace_lock); | 3795 | mutex_lock(&ftrace_lock); |
3152 | 3796 | ||
3797 | if (unlikely(ftrace_disabled)) | ||
3798 | goto out_unlock; | ||
3799 | |||
3153 | ret = __register_ftrace_function(ops); | 3800 | ret = __register_ftrace_function(ops); |
3154 | ftrace_startup(0); | 3801 | if (!ret) |
3802 | ftrace_startup(ops, 0); | ||
3155 | 3803 | ||
3804 | |||
3805 | out_unlock: | ||
3156 | mutex_unlock(&ftrace_lock); | 3806 | mutex_unlock(&ftrace_lock); |
3157 | return ret; | 3807 | return ret; |
3158 | } | 3808 | } |
3809 | EXPORT_SYMBOL_GPL(register_ftrace_function); | ||
3159 | 3810 | ||
3160 | /** | 3811 | /** |
3161 | * unregister_ftrace_function - unregister a function for profiling. | 3812 | * unregister_ftrace_function - unregister a function for profiling. |
@@ -3169,25 +3820,27 @@ int unregister_ftrace_function(struct ftrace_ops *ops) | |||
3169 | 3820 | ||
3170 | mutex_lock(&ftrace_lock); | 3821 | mutex_lock(&ftrace_lock); |
3171 | ret = __unregister_ftrace_function(ops); | 3822 | ret = __unregister_ftrace_function(ops); |
3172 | ftrace_shutdown(0); | 3823 | if (!ret) |
3824 | ftrace_shutdown(ops, 0); | ||
3173 | mutex_unlock(&ftrace_lock); | 3825 | mutex_unlock(&ftrace_lock); |
3174 | 3826 | ||
3175 | return ret; | 3827 | return ret; |
3176 | } | 3828 | } |
3829 | EXPORT_SYMBOL_GPL(unregister_ftrace_function); | ||
3177 | 3830 | ||
3178 | int | 3831 | int |
3179 | ftrace_enable_sysctl(struct ctl_table *table, int write, | 3832 | ftrace_enable_sysctl(struct ctl_table *table, int write, |
3180 | void __user *buffer, size_t *lenp, | 3833 | void __user *buffer, size_t *lenp, |
3181 | loff_t *ppos) | 3834 | loff_t *ppos) |
3182 | { | 3835 | { |
3183 | int ret; | 3836 | int ret = -ENODEV; |
3184 | |||
3185 | if (unlikely(ftrace_disabled)) | ||
3186 | return -ENODEV; | ||
3187 | 3837 | ||
3188 | mutex_lock(&ftrace_lock); | 3838 | mutex_lock(&ftrace_lock); |
3189 | 3839 | ||
3190 | ret = proc_dointvec(table, write, buffer, lenp, ppos); | 3840 | if (unlikely(ftrace_disabled)) |
3841 | goto out; | ||
3842 | |||
3843 | ret = proc_dointvec(table, write, buffer, lenp, ppos); | ||
3191 | 3844 | ||
3192 | if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) | 3845 | if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) |
3193 | goto out; | 3846 | goto out; |
@@ -3199,11 +3852,11 @@ ftrace_enable_sysctl(struct ctl_table *table, int write, | |||
3199 | ftrace_startup_sysctl(); | 3852 | ftrace_startup_sysctl(); |
3200 | 3853 | ||
3201 | /* we are starting ftrace again */ | 3854 | /* we are starting ftrace again */ |
3202 | if (ftrace_list != &ftrace_list_end) { | 3855 | if (ftrace_ops_list != &ftrace_list_end) { |
3203 | if (ftrace_list->next == &ftrace_list_end) | 3856 | if (ftrace_ops_list->next == &ftrace_list_end) |
3204 | ftrace_trace_function = ftrace_list->func; | 3857 | ftrace_trace_function = ftrace_ops_list->func; |
3205 | else | 3858 | else |
3206 | ftrace_trace_function = ftrace_list_func; | 3859 | ftrace_trace_function = ftrace_ops_list_func; |
3207 | } | 3860 | } |
3208 | 3861 | ||
3209 | } else { | 3862 | } else { |
@@ -3392,7 +4045,7 @@ int register_ftrace_graph(trace_func_graph_ret_t retfunc, | |||
3392 | ftrace_graph_return = retfunc; | 4045 | ftrace_graph_return = retfunc; |
3393 | ftrace_graph_entry = entryfunc; | 4046 | ftrace_graph_entry = entryfunc; |
3394 | 4047 | ||
3395 | ftrace_startup(FTRACE_START_FUNC_RET); | 4048 | ftrace_startup(&global_ops, FTRACE_START_FUNC_RET); |
3396 | 4049 | ||
3397 | out: | 4050 | out: |
3398 | mutex_unlock(&ftrace_lock); | 4051 | mutex_unlock(&ftrace_lock); |
@@ -3409,7 +4062,7 @@ void unregister_ftrace_graph(void) | |||
3409 | ftrace_graph_active--; | 4062 | ftrace_graph_active--; |
3410 | ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; | 4063 | ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; |
3411 | ftrace_graph_entry = ftrace_graph_entry_stub; | 4064 | ftrace_graph_entry = ftrace_graph_entry_stub; |
3412 | ftrace_shutdown(FTRACE_STOP_FUNC_RET); | 4065 | ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET); |
3413 | unregister_pm_notifier(&ftrace_suspend_notifier); | 4066 | unregister_pm_notifier(&ftrace_suspend_notifier); |
3414 | unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); | 4067 | unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); |
3415 | 4068 | ||
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 1cb49be7c7fb..ee9c921d7f21 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
@@ -2014,9 +2014,10 @@ enum print_line_t print_trace_line(struct trace_iterator *iter) | |||
2014 | { | 2014 | { |
2015 | enum print_line_t ret; | 2015 | enum print_line_t ret; |
2016 | 2016 | ||
2017 | if (iter->lost_events) | 2017 | if (iter->lost_events && |
2018 | trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", | 2018 | !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", |
2019 | iter->cpu, iter->lost_events); | 2019 | iter->cpu, iter->lost_events)) |
2020 | return TRACE_TYPE_PARTIAL_LINE; | ||
2020 | 2021 | ||
2021 | if (iter->trace && iter->trace->print_line) { | 2022 | if (iter->trace && iter->trace->print_line) { |
2022 | ret = iter->trace->print_line(iter); | 2023 | ret = iter->trace->print_line(iter); |
@@ -3230,6 +3231,14 @@ waitagain: | |||
3230 | 3231 | ||
3231 | if (iter->seq.len >= cnt) | 3232 | if (iter->seq.len >= cnt) |
3232 | break; | 3233 | break; |
3234 | |||
3235 | /* | ||
3236 | * Setting the full flag means we reached the trace_seq buffer | ||
3237 | * size and we should leave by partial output condition above. | ||
3238 | * One of the trace_seq_* functions is not used properly. | ||
3239 | */ | ||
3240 | WARN_ONCE(iter->seq.full, "full flag set for trace type %d", | ||
3241 | iter->ent->type); | ||
3233 | } | 3242 | } |
3234 | trace_access_unlock(iter->cpu_file); | 3243 | trace_access_unlock(iter->cpu_file); |
3235 | trace_event_read_unlock(); | 3244 | trace_event_read_unlock(); |
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 5e9dfc6286dd..6b69c4bd306f 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
@@ -419,6 +419,8 @@ extern void trace_find_cmdline(int pid, char comm[]); | |||
419 | extern unsigned long ftrace_update_tot_cnt; | 419 | extern unsigned long ftrace_update_tot_cnt; |
420 | #define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func | 420 | #define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func |
421 | extern int DYN_FTRACE_TEST_NAME(void); | 421 | extern int DYN_FTRACE_TEST_NAME(void); |
422 | #define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2 | ||
423 | extern int DYN_FTRACE_TEST_NAME2(void); | ||
422 | #endif | 424 | #endif |
423 | 425 | ||
424 | extern int ring_buffer_expanded; | 426 | extern int ring_buffer_expanded; |
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 16aee4d44e8f..8d0e1cc4e974 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c | |||
@@ -149,11 +149,13 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip) | |||
149 | static struct ftrace_ops trace_ops __read_mostly = | 149 | static struct ftrace_ops trace_ops __read_mostly = |
150 | { | 150 | { |
151 | .func = function_trace_call, | 151 | .func = function_trace_call, |
152 | .flags = FTRACE_OPS_FL_GLOBAL, | ||
152 | }; | 153 | }; |
153 | 154 | ||
154 | static struct ftrace_ops trace_stack_ops __read_mostly = | 155 | static struct ftrace_ops trace_stack_ops __read_mostly = |
155 | { | 156 | { |
156 | .func = function_stack_trace_call, | 157 | .func = function_stack_trace_call, |
158 | .flags = FTRACE_OPS_FL_GLOBAL, | ||
157 | }; | 159 | }; |
158 | 160 | ||
159 | /* Our two options */ | 161 | /* Our two options */ |
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c index a4969b47afc1..c77424be284d 100644 --- a/kernel/trace/trace_irqsoff.c +++ b/kernel/trace/trace_irqsoff.c | |||
@@ -153,6 +153,7 @@ irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip) | |||
153 | static struct ftrace_ops trace_ops __read_mostly = | 153 | static struct ftrace_ops trace_ops __read_mostly = |
154 | { | 154 | { |
155 | .func = irqsoff_tracer_call, | 155 | .func = irqsoff_tracer_call, |
156 | .flags = FTRACE_OPS_FL_GLOBAL, | ||
156 | }; | 157 | }; |
157 | #endif /* CONFIG_FUNCTION_TRACER */ | 158 | #endif /* CONFIG_FUNCTION_TRACER */ |
158 | 159 | ||
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index 456be9063c2d..cf535ccedc86 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c | |||
@@ -830,6 +830,9 @@ EXPORT_SYMBOL_GPL(unregister_ftrace_event); | |||
830 | enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags, | 830 | enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags, |
831 | struct trace_event *event) | 831 | struct trace_event *event) |
832 | { | 832 | { |
833 | if (!trace_seq_printf(&iter->seq, "type: %d\n", iter->ent->type)) | ||
834 | return TRACE_TYPE_PARTIAL_LINE; | ||
835 | |||
833 | return TRACE_TYPE_HANDLED; | 836 | return TRACE_TYPE_HANDLED; |
834 | } | 837 | } |
835 | 838 | ||
diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c index 2547d8813cf0..dff763b7baf1 100644 --- a/kernel/trace/trace_printk.c +++ b/kernel/trace/trace_printk.c | |||
@@ -32,7 +32,7 @@ static DEFINE_MUTEX(btrace_mutex); | |||
32 | 32 | ||
33 | struct trace_bprintk_fmt { | 33 | struct trace_bprintk_fmt { |
34 | struct list_head list; | 34 | struct list_head list; |
35 | char fmt[0]; | 35 | const char *fmt; |
36 | }; | 36 | }; |
37 | 37 | ||
38 | static inline struct trace_bprintk_fmt *lookup_format(const char *fmt) | 38 | static inline struct trace_bprintk_fmt *lookup_format(const char *fmt) |
@@ -49,6 +49,7 @@ static | |||
49 | void hold_module_trace_bprintk_format(const char **start, const char **end) | 49 | void hold_module_trace_bprintk_format(const char **start, const char **end) |
50 | { | 50 | { |
51 | const char **iter; | 51 | const char **iter; |
52 | char *fmt; | ||
52 | 53 | ||
53 | mutex_lock(&btrace_mutex); | 54 | mutex_lock(&btrace_mutex); |
54 | for (iter = start; iter < end; iter++) { | 55 | for (iter = start; iter < end; iter++) { |
@@ -58,14 +59,18 @@ void hold_module_trace_bprintk_format(const char **start, const char **end) | |||
58 | continue; | 59 | continue; |
59 | } | 60 | } |
60 | 61 | ||
61 | tb_fmt = kmalloc(offsetof(struct trace_bprintk_fmt, fmt) | 62 | tb_fmt = kmalloc(sizeof(*tb_fmt), GFP_KERNEL); |
62 | + strlen(*iter) + 1, GFP_KERNEL); | 63 | if (tb_fmt) |
63 | if (tb_fmt) { | 64 | fmt = kmalloc(strlen(*iter) + 1, GFP_KERNEL); |
65 | if (tb_fmt && fmt) { | ||
64 | list_add_tail(&tb_fmt->list, &trace_bprintk_fmt_list); | 66 | list_add_tail(&tb_fmt->list, &trace_bprintk_fmt_list); |
65 | strcpy(tb_fmt->fmt, *iter); | 67 | strcpy(fmt, *iter); |
68 | tb_fmt->fmt = fmt; | ||
66 | *iter = tb_fmt->fmt; | 69 | *iter = tb_fmt->fmt; |
67 | } else | 70 | } else { |
71 | kfree(tb_fmt); | ||
68 | *iter = NULL; | 72 | *iter = NULL; |
73 | } | ||
69 | } | 74 | } |
70 | mutex_unlock(&btrace_mutex); | 75 | mutex_unlock(&btrace_mutex); |
71 | } | 76 | } |
@@ -84,6 +89,76 @@ static int module_trace_bprintk_format_notify(struct notifier_block *self, | |||
84 | return 0; | 89 | return 0; |
85 | } | 90 | } |
86 | 91 | ||
92 | /* | ||
93 | * The debugfs/tracing/printk_formats file maps the addresses with | ||
94 | * the ASCII formats that are used in the bprintk events in the | ||
95 | * buffer. For userspace tools to be able to decode the events from | ||
96 | * the buffer, they need to be able to map the address with the format. | ||
97 | * | ||
98 | * The addresses of the bprintk formats are in their own section | ||
99 | * __trace_printk_fmt. But for modules we copy them into a link list. | ||
100 | * The code to print the formats and their addresses passes around the | ||
101 | * address of the fmt string. If the fmt address passed into the seq | ||
102 | * functions is within the kernel core __trace_printk_fmt section, then | ||
103 | * it simply uses the next pointer in the list. | ||
104 | * | ||
105 | * When the fmt pointer is outside the kernel core __trace_printk_fmt | ||
106 | * section, then we need to read the link list pointers. The trick is | ||
107 | * we pass the address of the string to the seq function just like | ||
108 | * we do for the kernel core formats. To get back the structure that | ||
109 | * holds the format, we simply use containerof() and then go to the | ||
110 | * next format in the list. | ||
111 | */ | ||
112 | static const char ** | ||
113 | find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos) | ||
114 | { | ||
115 | struct trace_bprintk_fmt *mod_fmt; | ||
116 | |||
117 | if (list_empty(&trace_bprintk_fmt_list)) | ||
118 | return NULL; | ||
119 | |||
120 | /* | ||
121 | * v will point to the address of the fmt record from t_next | ||
122 | * v will be NULL from t_start. | ||
123 | * If this is the first pointer or called from start | ||
124 | * then we need to walk the list. | ||
125 | */ | ||
126 | if (!v || start_index == *pos) { | ||
127 | struct trace_bprintk_fmt *p; | ||
128 | |||
129 | /* search the module list */ | ||
130 | list_for_each_entry(p, &trace_bprintk_fmt_list, list) { | ||
131 | if (start_index == *pos) | ||
132 | return &p->fmt; | ||
133 | start_index++; | ||
134 | } | ||
135 | /* pos > index */ | ||
136 | return NULL; | ||
137 | } | ||
138 | |||
139 | /* | ||
140 | * v points to the address of the fmt field in the mod list | ||
141 | * structure that holds the module print format. | ||
142 | */ | ||
143 | mod_fmt = container_of(v, typeof(*mod_fmt), fmt); | ||
144 | if (mod_fmt->list.next == &trace_bprintk_fmt_list) | ||
145 | return NULL; | ||
146 | |||
147 | mod_fmt = container_of(mod_fmt->list.next, typeof(*mod_fmt), list); | ||
148 | |||
149 | return &mod_fmt->fmt; | ||
150 | } | ||
151 | |||
152 | static void format_mod_start(void) | ||
153 | { | ||
154 | mutex_lock(&btrace_mutex); | ||
155 | } | ||
156 | |||
157 | static void format_mod_stop(void) | ||
158 | { | ||
159 | mutex_unlock(&btrace_mutex); | ||
160 | } | ||
161 | |||
87 | #else /* !CONFIG_MODULES */ | 162 | #else /* !CONFIG_MODULES */ |
88 | __init static int | 163 | __init static int |
89 | module_trace_bprintk_format_notify(struct notifier_block *self, | 164 | module_trace_bprintk_format_notify(struct notifier_block *self, |
@@ -91,6 +166,13 @@ module_trace_bprintk_format_notify(struct notifier_block *self, | |||
91 | { | 166 | { |
92 | return 0; | 167 | return 0; |
93 | } | 168 | } |
169 | static inline const char ** | ||
170 | find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos) | ||
171 | { | ||
172 | return NULL; | ||
173 | } | ||
174 | static inline void format_mod_start(void) { } | ||
175 | static inline void format_mod_stop(void) { } | ||
94 | #endif /* CONFIG_MODULES */ | 176 | #endif /* CONFIG_MODULES */ |
95 | 177 | ||
96 | 178 | ||
@@ -153,20 +235,33 @@ int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap) | |||
153 | } | 235 | } |
154 | EXPORT_SYMBOL_GPL(__ftrace_vprintk); | 236 | EXPORT_SYMBOL_GPL(__ftrace_vprintk); |
155 | 237 | ||
238 | static const char **find_next(void *v, loff_t *pos) | ||
239 | { | ||
240 | const char **fmt = v; | ||
241 | int start_index; | ||
242 | |||
243 | if (!fmt) | ||
244 | fmt = __start___trace_bprintk_fmt + *pos; | ||
245 | |||
246 | start_index = __stop___trace_bprintk_fmt - __start___trace_bprintk_fmt; | ||
247 | |||
248 | if (*pos < start_index) | ||
249 | return fmt; | ||
250 | |||
251 | return find_next_mod_format(start_index, v, fmt, pos); | ||
252 | } | ||
253 | |||
156 | static void * | 254 | static void * |
157 | t_start(struct seq_file *m, loff_t *pos) | 255 | t_start(struct seq_file *m, loff_t *pos) |
158 | { | 256 | { |
159 | const char **fmt = __start___trace_bprintk_fmt + *pos; | 257 | format_mod_start(); |
160 | 258 | return find_next(NULL, pos); | |
161 | if ((unsigned long)fmt >= (unsigned long)__stop___trace_bprintk_fmt) | ||
162 | return NULL; | ||
163 | return fmt; | ||
164 | } | 259 | } |
165 | 260 | ||
166 | static void *t_next(struct seq_file *m, void * v, loff_t *pos) | 261 | static void *t_next(struct seq_file *m, void * v, loff_t *pos) |
167 | { | 262 | { |
168 | (*pos)++; | 263 | (*pos)++; |
169 | return t_start(m, pos); | 264 | return find_next(v, pos); |
170 | } | 265 | } |
171 | 266 | ||
172 | static int t_show(struct seq_file *m, void *v) | 267 | static int t_show(struct seq_file *m, void *v) |
@@ -205,6 +300,7 @@ static int t_show(struct seq_file *m, void *v) | |||
205 | 300 | ||
206 | static void t_stop(struct seq_file *m, void *p) | 301 | static void t_stop(struct seq_file *m, void *p) |
207 | { | 302 | { |
303 | format_mod_stop(); | ||
208 | } | 304 | } |
209 | 305 | ||
210 | static const struct seq_operations show_format_seq_ops = { | 306 | static const struct seq_operations show_format_seq_ops = { |
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 7319559ed59f..f029dd4fd2ca 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c | |||
@@ -129,6 +129,7 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip) | |||
129 | static struct ftrace_ops trace_ops __read_mostly = | 129 | static struct ftrace_ops trace_ops __read_mostly = |
130 | { | 130 | { |
131 | .func = wakeup_tracer_call, | 131 | .func = wakeup_tracer_call, |
132 | .flags = FTRACE_OPS_FL_GLOBAL, | ||
132 | }; | 133 | }; |
133 | #endif /* CONFIG_FUNCTION_TRACER */ | 134 | #endif /* CONFIG_FUNCTION_TRACER */ |
134 | 135 | ||
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index 659732eba07c..288541f977fb 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c | |||
@@ -101,6 +101,206 @@ static inline void warn_failed_init_tracer(struct tracer *trace, int init_ret) | |||
101 | 101 | ||
102 | #ifdef CONFIG_DYNAMIC_FTRACE | 102 | #ifdef CONFIG_DYNAMIC_FTRACE |
103 | 103 | ||
104 | static int trace_selftest_test_probe1_cnt; | ||
105 | static void trace_selftest_test_probe1_func(unsigned long ip, | ||
106 | unsigned long pip) | ||
107 | { | ||
108 | trace_selftest_test_probe1_cnt++; | ||
109 | } | ||
110 | |||
111 | static int trace_selftest_test_probe2_cnt; | ||
112 | static void trace_selftest_test_probe2_func(unsigned long ip, | ||
113 | unsigned long pip) | ||
114 | { | ||
115 | trace_selftest_test_probe2_cnt++; | ||
116 | } | ||
117 | |||
118 | static int trace_selftest_test_probe3_cnt; | ||
119 | static void trace_selftest_test_probe3_func(unsigned long ip, | ||
120 | unsigned long pip) | ||
121 | { | ||
122 | trace_selftest_test_probe3_cnt++; | ||
123 | } | ||
124 | |||
125 | static int trace_selftest_test_global_cnt; | ||
126 | static void trace_selftest_test_global_func(unsigned long ip, | ||
127 | unsigned long pip) | ||
128 | { | ||
129 | trace_selftest_test_global_cnt++; | ||
130 | } | ||
131 | |||
132 | static int trace_selftest_test_dyn_cnt; | ||
133 | static void trace_selftest_test_dyn_func(unsigned long ip, | ||
134 | unsigned long pip) | ||
135 | { | ||
136 | trace_selftest_test_dyn_cnt++; | ||
137 | } | ||
138 | |||
139 | static struct ftrace_ops test_probe1 = { | ||
140 | .func = trace_selftest_test_probe1_func, | ||
141 | }; | ||
142 | |||
143 | static struct ftrace_ops test_probe2 = { | ||
144 | .func = trace_selftest_test_probe2_func, | ||
145 | }; | ||
146 | |||
147 | static struct ftrace_ops test_probe3 = { | ||
148 | .func = trace_selftest_test_probe3_func, | ||
149 | }; | ||
150 | |||
151 | static struct ftrace_ops test_global = { | ||
152 | .func = trace_selftest_test_global_func, | ||
153 | .flags = FTRACE_OPS_FL_GLOBAL, | ||
154 | }; | ||
155 | |||
156 | static void print_counts(void) | ||
157 | { | ||
158 | printk("(%d %d %d %d %d) ", | ||
159 | trace_selftest_test_probe1_cnt, | ||
160 | trace_selftest_test_probe2_cnt, | ||
161 | trace_selftest_test_probe3_cnt, | ||
162 | trace_selftest_test_global_cnt, | ||
163 | trace_selftest_test_dyn_cnt); | ||
164 | } | ||
165 | |||
166 | static void reset_counts(void) | ||
167 | { | ||
168 | trace_selftest_test_probe1_cnt = 0; | ||
169 | trace_selftest_test_probe2_cnt = 0; | ||
170 | trace_selftest_test_probe3_cnt = 0; | ||
171 | trace_selftest_test_global_cnt = 0; | ||
172 | trace_selftest_test_dyn_cnt = 0; | ||
173 | } | ||
174 | |||
175 | static int trace_selftest_ops(int cnt) | ||
176 | { | ||
177 | int save_ftrace_enabled = ftrace_enabled; | ||
178 | struct ftrace_ops *dyn_ops; | ||
179 | char *func1_name; | ||
180 | char *func2_name; | ||
181 | int len1; | ||
182 | int len2; | ||
183 | int ret = -1; | ||
184 | |||
185 | printk(KERN_CONT "PASSED\n"); | ||
186 | pr_info("Testing dynamic ftrace ops #%d: ", cnt); | ||
187 | |||
188 | ftrace_enabled = 1; | ||
189 | reset_counts(); | ||
190 | |||
191 | /* Handle PPC64 '.' name */ | ||
192 | func1_name = "*" __stringify(DYN_FTRACE_TEST_NAME); | ||
193 | func2_name = "*" __stringify(DYN_FTRACE_TEST_NAME2); | ||
194 | len1 = strlen(func1_name); | ||
195 | len2 = strlen(func2_name); | ||
196 | |||
197 | /* | ||
198 | * Probe 1 will trace function 1. | ||
199 | * Probe 2 will trace function 2. | ||
200 | * Probe 3 will trace functions 1 and 2. | ||
201 | */ | ||
202 | ftrace_set_filter(&test_probe1, func1_name, len1, 1); | ||
203 | ftrace_set_filter(&test_probe2, func2_name, len2, 1); | ||
204 | ftrace_set_filter(&test_probe3, func1_name, len1, 1); | ||
205 | ftrace_set_filter(&test_probe3, func2_name, len2, 0); | ||
206 | |||
207 | register_ftrace_function(&test_probe1); | ||
208 | register_ftrace_function(&test_probe2); | ||
209 | register_ftrace_function(&test_probe3); | ||
210 | register_ftrace_function(&test_global); | ||
211 | |||
212 | DYN_FTRACE_TEST_NAME(); | ||
213 | |||
214 | print_counts(); | ||
215 | |||
216 | if (trace_selftest_test_probe1_cnt != 1) | ||
217 | goto out; | ||
218 | if (trace_selftest_test_probe2_cnt != 0) | ||
219 | goto out; | ||
220 | if (trace_selftest_test_probe3_cnt != 1) | ||
221 | goto out; | ||
222 | if (trace_selftest_test_global_cnt == 0) | ||
223 | goto out; | ||
224 | |||
225 | DYN_FTRACE_TEST_NAME2(); | ||
226 | |||
227 | print_counts(); | ||
228 | |||
229 | if (trace_selftest_test_probe1_cnt != 1) | ||
230 | goto out; | ||
231 | if (trace_selftest_test_probe2_cnt != 1) | ||
232 | goto out; | ||
233 | if (trace_selftest_test_probe3_cnt != 2) | ||
234 | goto out; | ||
235 | |||
236 | /* Add a dynamic probe */ | ||
237 | dyn_ops = kzalloc(sizeof(*dyn_ops), GFP_KERNEL); | ||
238 | if (!dyn_ops) { | ||
239 | printk("MEMORY ERROR "); | ||
240 | goto out; | ||
241 | } | ||
242 | |||
243 | dyn_ops->func = trace_selftest_test_dyn_func; | ||
244 | |||
245 | register_ftrace_function(dyn_ops); | ||
246 | |||
247 | trace_selftest_test_global_cnt = 0; | ||
248 | |||
249 | DYN_FTRACE_TEST_NAME(); | ||
250 | |||
251 | print_counts(); | ||
252 | |||
253 | if (trace_selftest_test_probe1_cnt != 2) | ||
254 | goto out_free; | ||
255 | if (trace_selftest_test_probe2_cnt != 1) | ||
256 | goto out_free; | ||
257 | if (trace_selftest_test_probe3_cnt != 3) | ||
258 | goto out_free; | ||
259 | if (trace_selftest_test_global_cnt == 0) | ||
260 | goto out; | ||
261 | if (trace_selftest_test_dyn_cnt == 0) | ||
262 | goto out_free; | ||
263 | |||
264 | DYN_FTRACE_TEST_NAME2(); | ||
265 | |||
266 | print_counts(); | ||
267 | |||
268 | if (trace_selftest_test_probe1_cnt != 2) | ||
269 | goto out_free; | ||
270 | if (trace_selftest_test_probe2_cnt != 2) | ||
271 | goto out_free; | ||
272 | if (trace_selftest_test_probe3_cnt != 4) | ||
273 | goto out_free; | ||
274 | |||
275 | ret = 0; | ||
276 | out_free: | ||
277 | unregister_ftrace_function(dyn_ops); | ||
278 | kfree(dyn_ops); | ||
279 | |||
280 | out: | ||
281 | /* Purposely unregister in the same order */ | ||
282 | unregister_ftrace_function(&test_probe1); | ||
283 | unregister_ftrace_function(&test_probe2); | ||
284 | unregister_ftrace_function(&test_probe3); | ||
285 | unregister_ftrace_function(&test_global); | ||
286 | |||
287 | /* Make sure everything is off */ | ||
288 | reset_counts(); | ||
289 | DYN_FTRACE_TEST_NAME(); | ||
290 | DYN_FTRACE_TEST_NAME(); | ||
291 | |||
292 | if (trace_selftest_test_probe1_cnt || | ||
293 | trace_selftest_test_probe2_cnt || | ||
294 | trace_selftest_test_probe3_cnt || | ||
295 | trace_selftest_test_global_cnt || | ||
296 | trace_selftest_test_dyn_cnt) | ||
297 | ret = -1; | ||
298 | |||
299 | ftrace_enabled = save_ftrace_enabled; | ||
300 | |||
301 | return ret; | ||
302 | } | ||
303 | |||
104 | /* Test dynamic code modification and ftrace filters */ | 304 | /* Test dynamic code modification and ftrace filters */ |
105 | int trace_selftest_startup_dynamic_tracing(struct tracer *trace, | 305 | int trace_selftest_startup_dynamic_tracing(struct tracer *trace, |
106 | struct trace_array *tr, | 306 | struct trace_array *tr, |
@@ -131,7 +331,7 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace, | |||
131 | func_name = "*" __stringify(DYN_FTRACE_TEST_NAME); | 331 | func_name = "*" __stringify(DYN_FTRACE_TEST_NAME); |
132 | 332 | ||
133 | /* filter only on our function */ | 333 | /* filter only on our function */ |
134 | ftrace_set_filter(func_name, strlen(func_name), 1); | 334 | ftrace_set_global_filter(func_name, strlen(func_name), 1); |
135 | 335 | ||
136 | /* enable tracing */ | 336 | /* enable tracing */ |
137 | ret = tracer_init(trace, tr); | 337 | ret = tracer_init(trace, tr); |
@@ -166,22 +366,30 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace, | |||
166 | 366 | ||
167 | /* check the trace buffer */ | 367 | /* check the trace buffer */ |
168 | ret = trace_test_buffer(tr, &count); | 368 | ret = trace_test_buffer(tr, &count); |
169 | trace->reset(tr); | ||
170 | tracing_start(); | 369 | tracing_start(); |
171 | 370 | ||
172 | /* we should only have one item */ | 371 | /* we should only have one item */ |
173 | if (!ret && count != 1) { | 372 | if (!ret && count != 1) { |
373 | trace->reset(tr); | ||
174 | printk(KERN_CONT ".. filter failed count=%ld ..", count); | 374 | printk(KERN_CONT ".. filter failed count=%ld ..", count); |
175 | ret = -1; | 375 | ret = -1; |
176 | goto out; | 376 | goto out; |
177 | } | 377 | } |
178 | 378 | ||
379 | /* Test the ops with global tracing running */ | ||
380 | ret = trace_selftest_ops(1); | ||
381 | trace->reset(tr); | ||
382 | |||
179 | out: | 383 | out: |
180 | ftrace_enabled = save_ftrace_enabled; | 384 | ftrace_enabled = save_ftrace_enabled; |
181 | tracer_enabled = save_tracer_enabled; | 385 | tracer_enabled = save_tracer_enabled; |
182 | 386 | ||
183 | /* Enable tracing on all functions again */ | 387 | /* Enable tracing on all functions again */ |
184 | ftrace_set_filter(NULL, 0, 1); | 388 | ftrace_set_global_filter(NULL, 0, 1); |
389 | |||
390 | /* Test the ops with global tracing off */ | ||
391 | if (!ret) | ||
392 | ret = trace_selftest_ops(2); | ||
185 | 393 | ||
186 | return ret; | 394 | return ret; |
187 | } | 395 | } |
diff --git a/kernel/trace/trace_selftest_dynamic.c b/kernel/trace/trace_selftest_dynamic.c index 54dd77cce5bf..b4c475a0a48b 100644 --- a/kernel/trace/trace_selftest_dynamic.c +++ b/kernel/trace/trace_selftest_dynamic.c | |||
@@ -5,3 +5,9 @@ int DYN_FTRACE_TEST_NAME(void) | |||
5 | /* used to call mcount */ | 5 | /* used to call mcount */ |
6 | return 0; | 6 | return 0; |
7 | } | 7 | } |
8 | |||
9 | int DYN_FTRACE_TEST_NAME2(void) | ||
10 | { | ||
11 | /* used to call mcount */ | ||
12 | return 0; | ||
13 | } | ||
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c index 4c5dead0c239..b0b53b8e4c25 100644 --- a/kernel/trace/trace_stack.c +++ b/kernel/trace/trace_stack.c | |||
@@ -133,6 +133,7 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip) | |||
133 | static struct ftrace_ops trace_ops __read_mostly = | 133 | static struct ftrace_ops trace_ops __read_mostly = |
134 | { | 134 | { |
135 | .func = stack_trace_call, | 135 | .func = stack_trace_call, |
136 | .flags = FTRACE_OPS_FL_GLOBAL, | ||
136 | }; | 137 | }; |
137 | 138 | ||
138 | static ssize_t | 139 | static ssize_t |
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c index 68187af4889e..b219f1449c54 100644 --- a/kernel/tracepoint.c +++ b/kernel/tracepoint.c | |||
@@ -251,9 +251,9 @@ static void set_tracepoint(struct tracepoint_entry **entry, | |||
251 | { | 251 | { |
252 | WARN_ON(strcmp((*entry)->name, elem->name) != 0); | 252 | WARN_ON(strcmp((*entry)->name, elem->name) != 0); |
253 | 253 | ||
254 | if (elem->regfunc && !elem->state && active) | 254 | if (elem->regfunc && !jump_label_enabled(&elem->key) && active) |
255 | elem->regfunc(); | 255 | elem->regfunc(); |
256 | else if (elem->unregfunc && elem->state && !active) | 256 | else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active) |
257 | elem->unregfunc(); | 257 | elem->unregfunc(); |
258 | 258 | ||
259 | /* | 259 | /* |
@@ -264,13 +264,10 @@ static void set_tracepoint(struct tracepoint_entry **entry, | |||
264 | * is used. | 264 | * is used. |
265 | */ | 265 | */ |
266 | rcu_assign_pointer(elem->funcs, (*entry)->funcs); | 266 | rcu_assign_pointer(elem->funcs, (*entry)->funcs); |
267 | if (!elem->state && active) { | 267 | if (active && !jump_label_enabled(&elem->key)) |
268 | jump_label_enable(&elem->state); | 268 | jump_label_inc(&elem->key); |
269 | elem->state = active; | 269 | else if (!active && jump_label_enabled(&elem->key)) |
270 | } else if (elem->state && !active) { | 270 | jump_label_dec(&elem->key); |
271 | jump_label_disable(&elem->state); | ||
272 | elem->state = active; | ||
273 | } | ||
274 | } | 271 | } |
275 | 272 | ||
276 | /* | 273 | /* |
@@ -281,13 +278,11 @@ static void set_tracepoint(struct tracepoint_entry **entry, | |||
281 | */ | 278 | */ |
282 | static void disable_tracepoint(struct tracepoint *elem) | 279 | static void disable_tracepoint(struct tracepoint *elem) |
283 | { | 280 | { |
284 | if (elem->unregfunc && elem->state) | 281 | if (elem->unregfunc && jump_label_enabled(&elem->key)) |
285 | elem->unregfunc(); | 282 | elem->unregfunc(); |
286 | 283 | ||
287 | if (elem->state) { | 284 | if (jump_label_enabled(&elem->key)) |
288 | jump_label_disable(&elem->state); | 285 | jump_label_dec(&elem->key); |
289 | elem->state = 0; | ||
290 | } | ||
291 | rcu_assign_pointer(elem->funcs, NULL); | 286 | rcu_assign_pointer(elem->funcs, NULL); |
292 | } | 287 | } |
293 | 288 | ||
diff --git a/scripts/Makefile.build b/scripts/Makefile.build index d5f925abe4d2..6165622c3e29 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build | |||
@@ -244,14 +244,19 @@ endif | |||
244 | 244 | ||
245 | ifdef CONFIG_FTRACE_MCOUNT_RECORD | 245 | ifdef CONFIG_FTRACE_MCOUNT_RECORD |
246 | ifdef BUILD_C_RECORDMCOUNT | 246 | ifdef BUILD_C_RECORDMCOUNT |
247 | ifeq ("$(origin RECORDMCOUNT_WARN)", "command line") | ||
248 | RECORDMCOUNT_FLAGS = -w | ||
249 | endif | ||
247 | # Due to recursion, we must skip empty.o. | 250 | # Due to recursion, we must skip empty.o. |
248 | # The empty.o file is created in the make process in order to determine | 251 | # The empty.o file is created in the make process in order to determine |
249 | # the target endianness and word size. It is made before all other C | 252 | # the target endianness and word size. It is made before all other C |
250 | # files, including recordmcount. | 253 | # files, including recordmcount. |
251 | sub_cmd_record_mcount = \ | 254 | sub_cmd_record_mcount = \ |
252 | if [ $(@) != "scripts/mod/empty.o" ]; then \ | 255 | if [ $(@) != "scripts/mod/empty.o" ]; then \ |
253 | $(objtree)/scripts/recordmcount "$(@)"; \ | 256 | $(objtree)/scripts/recordmcount $(RECORDMCOUNT_FLAGS) "$(@)"; \ |
254 | fi; | 257 | fi; |
258 | recordmcount_source := $(srctree)/scripts/recordmcount.c \ | ||
259 | $(srctree)/scripts/recordmcount.h | ||
255 | else | 260 | else |
256 | sub_cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \ | 261 | sub_cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \ |
257 | "$(if $(CONFIG_CPU_BIG_ENDIAN),big,little)" \ | 262 | "$(if $(CONFIG_CPU_BIG_ENDIAN),big,little)" \ |
@@ -259,6 +264,7 @@ sub_cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH | |||
259 | "$(OBJDUMP)" "$(OBJCOPY)" "$(CC) $(KBUILD_CFLAGS)" \ | 264 | "$(OBJDUMP)" "$(OBJCOPY)" "$(CC) $(KBUILD_CFLAGS)" \ |
260 | "$(LD)" "$(NM)" "$(RM)" "$(MV)" \ | 265 | "$(LD)" "$(NM)" "$(RM)" "$(MV)" \ |
261 | "$(if $(part-of-module),1,0)" "$(@)"; | 266 | "$(if $(part-of-module),1,0)" "$(@)"; |
267 | recordmcount_source := $(srctree)/scripts/recordmcount.pl | ||
262 | endif | 268 | endif |
263 | cmd_record_mcount = \ | 269 | cmd_record_mcount = \ |
264 | if [ "$(findstring -pg,$(_c_flags))" = "-pg" ]; then \ | 270 | if [ "$(findstring -pg,$(_c_flags))" = "-pg" ]; then \ |
@@ -279,13 +285,13 @@ define rule_cc_o_c | |||
279 | endef | 285 | endef |
280 | 286 | ||
281 | # Built-in and composite module parts | 287 | # Built-in and composite module parts |
282 | $(obj)/%.o: $(src)/%.c FORCE | 288 | $(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE |
283 | $(call cmd,force_checksrc) | 289 | $(call cmd,force_checksrc) |
284 | $(call if_changed_rule,cc_o_c) | 290 | $(call if_changed_rule,cc_o_c) |
285 | 291 | ||
286 | # Single-part modules are special since we need to mark them in $(MODVERDIR) | 292 | # Single-part modules are special since we need to mark them in $(MODVERDIR) |
287 | 293 | ||
288 | $(single-used-m): $(obj)/%.o: $(src)/%.c FORCE | 294 | $(single-used-m): $(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE |
289 | $(call cmd,force_checksrc) | 295 | $(call cmd,force_checksrc) |
290 | $(call if_changed_rule,cc_o_c) | 296 | $(call if_changed_rule,cc_o_c) |
291 | @{ echo $(@:.o=.ko); echo $@; } > $(MODVERDIR)/$(@F:.o=.mod) | 297 | @{ echo $(@:.o=.ko); echo $@; } > $(MODVERDIR)/$(@F:.o=.mod) |
diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index f9f6f52db772..ee52cb8e17ad 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c | |||
@@ -24,6 +24,7 @@ | |||
24 | #include <sys/types.h> | 24 | #include <sys/types.h> |
25 | #include <sys/mman.h> | 25 | #include <sys/mman.h> |
26 | #include <sys/stat.h> | 26 | #include <sys/stat.h> |
27 | #include <getopt.h> | ||
27 | #include <elf.h> | 28 | #include <elf.h> |
28 | #include <fcntl.h> | 29 | #include <fcntl.h> |
29 | #include <setjmp.h> | 30 | #include <setjmp.h> |
@@ -39,6 +40,7 @@ static char gpfx; /* prefix for global symbol name (sometimes '_') */ | |||
39 | static struct stat sb; /* Remember .st_size, etc. */ | 40 | static struct stat sb; /* Remember .st_size, etc. */ |
40 | static jmp_buf jmpenv; /* setjmp/longjmp per-file error escape */ | 41 | static jmp_buf jmpenv; /* setjmp/longjmp per-file error escape */ |
41 | static const char *altmcount; /* alternate mcount symbol name */ | 42 | static const char *altmcount; /* alternate mcount symbol name */ |
43 | static int warn_on_notrace_sect; /* warn when section has mcount not being recorded */ | ||
42 | 44 | ||
43 | /* setjmp() return values */ | 45 | /* setjmp() return values */ |
44 | enum { | 46 | enum { |
@@ -78,7 +80,7 @@ static off_t | |||
78 | ulseek(int const fd, off_t const offset, int const whence) | 80 | ulseek(int const fd, off_t const offset, int const whence) |
79 | { | 81 | { |
80 | off_t const w = lseek(fd, offset, whence); | 82 | off_t const w = lseek(fd, offset, whence); |
81 | if ((off_t)-1 == w) { | 83 | if (w == (off_t)-1) { |
82 | perror("lseek"); | 84 | perror("lseek"); |
83 | fail_file(); | 85 | fail_file(); |
84 | } | 86 | } |
@@ -111,13 +113,41 @@ static void * | |||
111 | umalloc(size_t size) | 113 | umalloc(size_t size) |
112 | { | 114 | { |
113 | void *const addr = malloc(size); | 115 | void *const addr = malloc(size); |
114 | if (0 == addr) { | 116 | if (addr == 0) { |
115 | fprintf(stderr, "malloc failed: %zu bytes\n", size); | 117 | fprintf(stderr, "malloc failed: %zu bytes\n", size); |
116 | fail_file(); | 118 | fail_file(); |
117 | } | 119 | } |
118 | return addr; | 120 | return addr; |
119 | } | 121 | } |
120 | 122 | ||
123 | static unsigned char ideal_nop5_x86_64[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 }; | ||
124 | static unsigned char ideal_nop5_x86_32[5] = { 0x3e, 0x8d, 0x74, 0x26, 0x00 }; | ||
125 | static unsigned char *ideal_nop; | ||
126 | |||
127 | static char rel_type_nop; | ||
128 | |||
129 | static int (*make_nop)(void *map, size_t const offset); | ||
130 | |||
131 | static int make_nop_x86(void *map, size_t const offset) | ||
132 | { | ||
133 | uint32_t *ptr; | ||
134 | unsigned char *op; | ||
135 | |||
136 | /* Confirm we have 0xe8 0x0 0x0 0x0 0x0 */ | ||
137 | ptr = map + offset; | ||
138 | if (*ptr != 0) | ||
139 | return -1; | ||
140 | |||
141 | op = map + offset - 1; | ||
142 | if (*op != 0xe8) | ||
143 | return -1; | ||
144 | |||
145 | /* convert to nop */ | ||
146 | ulseek(fd_map, offset - 1, SEEK_SET); | ||
147 | uwrite(fd_map, ideal_nop, 5); | ||
148 | return 0; | ||
149 | } | ||
150 | |||
121 | /* | 151 | /* |
122 | * Get the whole file as a programming convenience in order to avoid | 152 | * Get the whole file as a programming convenience in order to avoid |
123 | * malloc+lseek+read+free of many pieces. If successful, then mmap | 153 | * malloc+lseek+read+free of many pieces. If successful, then mmap |
@@ -136,7 +166,7 @@ static void *mmap_file(char const *fname) | |||
136 | void *addr; | 166 | void *addr; |
137 | 167 | ||
138 | fd_map = open(fname, O_RDWR); | 168 | fd_map = open(fname, O_RDWR); |
139 | if (0 > fd_map || 0 > fstat(fd_map, &sb)) { | 169 | if (fd_map < 0 || fstat(fd_map, &sb) < 0) { |
140 | perror(fname); | 170 | perror(fname); |
141 | fail_file(); | 171 | fail_file(); |
142 | } | 172 | } |
@@ -147,7 +177,7 @@ static void *mmap_file(char const *fname) | |||
147 | addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE, | 177 | addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE, |
148 | fd_map, 0); | 178 | fd_map, 0); |
149 | mmap_failed = 0; | 179 | mmap_failed = 0; |
150 | if (MAP_FAILED == addr) { | 180 | if (addr == MAP_FAILED) { |
151 | mmap_failed = 1; | 181 | mmap_failed = 1; |
152 | addr = umalloc(sb.st_size); | 182 | addr = umalloc(sb.st_size); |
153 | uread(fd_map, addr, sb.st_size); | 183 | uread(fd_map, addr, sb.st_size); |
@@ -206,12 +236,13 @@ static uint32_t (*w2)(uint16_t); | |||
206 | static int | 236 | static int |
207 | is_mcounted_section_name(char const *const txtname) | 237 | is_mcounted_section_name(char const *const txtname) |
208 | { | 238 | { |
209 | return 0 == strcmp(".text", txtname) || | 239 | return strcmp(".text", txtname) == 0 || |
210 | 0 == strcmp(".ref.text", txtname) || | 240 | strcmp(".ref.text", txtname) == 0 || |
211 | 0 == strcmp(".sched.text", txtname) || | 241 | strcmp(".sched.text", txtname) == 0 || |
212 | 0 == strcmp(".spinlock.text", txtname) || | 242 | strcmp(".spinlock.text", txtname) == 0 || |
213 | 0 == strcmp(".irqentry.text", txtname) || | 243 | strcmp(".irqentry.text", txtname) == 0 || |
214 | 0 == strcmp(".text.unlikely", txtname); | 244 | strcmp(".kprobes.text", txtname) == 0 || |
245 | strcmp(".text.unlikely", txtname) == 0; | ||
215 | } | 246 | } |
216 | 247 | ||
217 | /* 32 bit and 64 bit are very similar */ | 248 | /* 32 bit and 64 bit are very similar */ |
@@ -264,43 +295,48 @@ do_file(char const *const fname) | |||
264 | w8 = w8nat; | 295 | w8 = w8nat; |
265 | switch (ehdr->e_ident[EI_DATA]) { | 296 | switch (ehdr->e_ident[EI_DATA]) { |
266 | static unsigned int const endian = 1; | 297 | static unsigned int const endian = 1; |
267 | default: { | 298 | default: |
268 | fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", | 299 | fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", |
269 | ehdr->e_ident[EI_DATA], fname); | 300 | ehdr->e_ident[EI_DATA], fname); |
270 | fail_file(); | 301 | fail_file(); |
271 | } break; | 302 | break; |
272 | case ELFDATA2LSB: { | 303 | case ELFDATA2LSB: |
273 | if (1 != *(unsigned char const *)&endian) { | 304 | if (*(unsigned char const *)&endian != 1) { |
274 | /* main() is big endian, file.o is little endian. */ | 305 | /* main() is big endian, file.o is little endian. */ |
275 | w = w4rev; | 306 | w = w4rev; |
276 | w2 = w2rev; | 307 | w2 = w2rev; |
277 | w8 = w8rev; | 308 | w8 = w8rev; |
278 | } | 309 | } |
279 | } break; | 310 | break; |
280 | case ELFDATA2MSB: { | 311 | case ELFDATA2MSB: |
281 | if (0 != *(unsigned char const *)&endian) { | 312 | if (*(unsigned char const *)&endian != 0) { |
282 | /* main() is little endian, file.o is big endian. */ | 313 | /* main() is little endian, file.o is big endian. */ |
283 | w = w4rev; | 314 | w = w4rev; |
284 | w2 = w2rev; | 315 | w2 = w2rev; |
285 | w8 = w8rev; | 316 | w8 = w8rev; |
286 | } | 317 | } |
287 | } break; | 318 | break; |
288 | } /* end switch */ | 319 | } /* end switch */ |
289 | if (0 != memcmp(ELFMAG, ehdr->e_ident, SELFMAG) | 320 | if (memcmp(ELFMAG, ehdr->e_ident, SELFMAG) != 0 |
290 | || ET_REL != w2(ehdr->e_type) | 321 | || w2(ehdr->e_type) != ET_REL |
291 | || EV_CURRENT != ehdr->e_ident[EI_VERSION]) { | 322 | || ehdr->e_ident[EI_VERSION] != EV_CURRENT) { |
292 | fprintf(stderr, "unrecognized ET_REL file %s\n", fname); | 323 | fprintf(stderr, "unrecognized ET_REL file %s\n", fname); |
293 | fail_file(); | 324 | fail_file(); |
294 | } | 325 | } |
295 | 326 | ||
296 | gpfx = 0; | 327 | gpfx = 0; |
297 | switch (w2(ehdr->e_machine)) { | 328 | switch (w2(ehdr->e_machine)) { |
298 | default: { | 329 | default: |
299 | fprintf(stderr, "unrecognized e_machine %d %s\n", | 330 | fprintf(stderr, "unrecognized e_machine %d %s\n", |
300 | w2(ehdr->e_machine), fname); | 331 | w2(ehdr->e_machine), fname); |
301 | fail_file(); | 332 | fail_file(); |
302 | } break; | 333 | break; |
303 | case EM_386: reltype = R_386_32; break; | 334 | case EM_386: |
335 | reltype = R_386_32; | ||
336 | make_nop = make_nop_x86; | ||
337 | ideal_nop = ideal_nop5_x86_32; | ||
338 | mcount_adjust_32 = -1; | ||
339 | break; | ||
304 | case EM_ARM: reltype = R_ARM_ABS32; | 340 | case EM_ARM: reltype = R_ARM_ABS32; |
305 | altmcount = "__gnu_mcount_nc"; | 341 | altmcount = "__gnu_mcount_nc"; |
306 | break; | 342 | break; |
@@ -311,67 +347,91 @@ do_file(char const *const fname) | |||
311 | case EM_S390: /* reltype: e_class */ gpfx = '_'; break; | 347 | case EM_S390: /* reltype: e_class */ gpfx = '_'; break; |
312 | case EM_SH: reltype = R_SH_DIR32; break; | 348 | case EM_SH: reltype = R_SH_DIR32; break; |
313 | case EM_SPARCV9: reltype = R_SPARC_64; gpfx = '_'; break; | 349 | case EM_SPARCV9: reltype = R_SPARC_64; gpfx = '_'; break; |
314 | case EM_X86_64: reltype = R_X86_64_64; break; | 350 | case EM_X86_64: |
351 | make_nop = make_nop_x86; | ||
352 | ideal_nop = ideal_nop5_x86_64; | ||
353 | reltype = R_X86_64_64; | ||
354 | mcount_adjust_64 = -1; | ||
355 | break; | ||
315 | } /* end switch */ | 356 | } /* end switch */ |
316 | 357 | ||
317 | switch (ehdr->e_ident[EI_CLASS]) { | 358 | switch (ehdr->e_ident[EI_CLASS]) { |
318 | default: { | 359 | default: |
319 | fprintf(stderr, "unrecognized ELF class %d %s\n", | 360 | fprintf(stderr, "unrecognized ELF class %d %s\n", |
320 | ehdr->e_ident[EI_CLASS], fname); | 361 | ehdr->e_ident[EI_CLASS], fname); |
321 | fail_file(); | 362 | fail_file(); |
322 | } break; | 363 | break; |
323 | case ELFCLASS32: { | 364 | case ELFCLASS32: |
324 | if (sizeof(Elf32_Ehdr) != w2(ehdr->e_ehsize) | 365 | if (w2(ehdr->e_ehsize) != sizeof(Elf32_Ehdr) |
325 | || sizeof(Elf32_Shdr) != w2(ehdr->e_shentsize)) { | 366 | || w2(ehdr->e_shentsize) != sizeof(Elf32_Shdr)) { |
326 | fprintf(stderr, | 367 | fprintf(stderr, |
327 | "unrecognized ET_REL file: %s\n", fname); | 368 | "unrecognized ET_REL file: %s\n", fname); |
328 | fail_file(); | 369 | fail_file(); |
329 | } | 370 | } |
330 | if (EM_S390 == w2(ehdr->e_machine)) | 371 | if (w2(ehdr->e_machine) == EM_S390) { |
331 | reltype = R_390_32; | 372 | reltype = R_390_32; |
332 | if (EM_MIPS == w2(ehdr->e_machine)) { | 373 | mcount_adjust_32 = -4; |
374 | } | ||
375 | if (w2(ehdr->e_machine) == EM_MIPS) { | ||
333 | reltype = R_MIPS_32; | 376 | reltype = R_MIPS_32; |
334 | is_fake_mcount32 = MIPS32_is_fake_mcount; | 377 | is_fake_mcount32 = MIPS32_is_fake_mcount; |
335 | } | 378 | } |
336 | do32(ehdr, fname, reltype); | 379 | do32(ehdr, fname, reltype); |
337 | } break; | 380 | break; |
338 | case ELFCLASS64: { | 381 | case ELFCLASS64: { |
339 | Elf64_Ehdr *const ghdr = (Elf64_Ehdr *)ehdr; | 382 | Elf64_Ehdr *const ghdr = (Elf64_Ehdr *)ehdr; |
340 | if (sizeof(Elf64_Ehdr) != w2(ghdr->e_ehsize) | 383 | if (w2(ghdr->e_ehsize) != sizeof(Elf64_Ehdr) |
341 | || sizeof(Elf64_Shdr) != w2(ghdr->e_shentsize)) { | 384 | || w2(ghdr->e_shentsize) != sizeof(Elf64_Shdr)) { |
342 | fprintf(stderr, | 385 | fprintf(stderr, |
343 | "unrecognized ET_REL file: %s\n", fname); | 386 | "unrecognized ET_REL file: %s\n", fname); |
344 | fail_file(); | 387 | fail_file(); |
345 | } | 388 | } |
346 | if (EM_S390 == w2(ghdr->e_machine)) | 389 | if (w2(ghdr->e_machine) == EM_S390) { |
347 | reltype = R_390_64; | 390 | reltype = R_390_64; |
348 | if (EM_MIPS == w2(ghdr->e_machine)) { | 391 | mcount_adjust_64 = -8; |
392 | } | ||
393 | if (w2(ghdr->e_machine) == EM_MIPS) { | ||
349 | reltype = R_MIPS_64; | 394 | reltype = R_MIPS_64; |
350 | Elf64_r_sym = MIPS64_r_sym; | 395 | Elf64_r_sym = MIPS64_r_sym; |
351 | Elf64_r_info = MIPS64_r_info; | 396 | Elf64_r_info = MIPS64_r_info; |
352 | is_fake_mcount64 = MIPS64_is_fake_mcount; | 397 | is_fake_mcount64 = MIPS64_is_fake_mcount; |
353 | } | 398 | } |
354 | do64(ghdr, fname, reltype); | 399 | do64(ghdr, fname, reltype); |
355 | } break; | 400 | break; |
401 | } | ||
356 | } /* end switch */ | 402 | } /* end switch */ |
357 | 403 | ||
358 | cleanup(); | 404 | cleanup(); |
359 | } | 405 | } |
360 | 406 | ||
361 | int | 407 | int |
362 | main(int argc, char const *argv[]) | 408 | main(int argc, char *argv[]) |
363 | { | 409 | { |
364 | const char ftrace[] = "/ftrace.o"; | 410 | const char ftrace[] = "/ftrace.o"; |
365 | int ftrace_size = sizeof(ftrace) - 1; | 411 | int ftrace_size = sizeof(ftrace) - 1; |
366 | int n_error = 0; /* gcc-4.3.0 false positive complaint */ | 412 | int n_error = 0; /* gcc-4.3.0 false positive complaint */ |
413 | int c; | ||
414 | int i; | ||
415 | |||
416 | while ((c = getopt(argc, argv, "w")) >= 0) { | ||
417 | switch (c) { | ||
418 | case 'w': | ||
419 | warn_on_notrace_sect = 1; | ||
420 | break; | ||
421 | default: | ||
422 | fprintf(stderr, "usage: recordmcount [-w] file.o...\n"); | ||
423 | return 0; | ||
424 | } | ||
425 | } | ||
367 | 426 | ||
368 | if (argc <= 1) { | 427 | if ((argc - optind) < 1) { |
369 | fprintf(stderr, "usage: recordmcount file.o...\n"); | 428 | fprintf(stderr, "usage: recordmcount [-w] file.o...\n"); |
370 | return 0; | 429 | return 0; |
371 | } | 430 | } |
372 | 431 | ||
373 | /* Process each file in turn, allowing deep failure. */ | 432 | /* Process each file in turn, allowing deep failure. */ |
374 | for (--argc, ++argv; 0 < argc; --argc, ++argv) { | 433 | for (i = optind; i < argc; i++) { |
434 | char *file = argv[i]; | ||
375 | int const sjval = setjmp(jmpenv); | 435 | int const sjval = setjmp(jmpenv); |
376 | int len; | 436 | int len; |
377 | 437 | ||
@@ -380,29 +440,29 @@ main(int argc, char const *argv[]) | |||
380 | * function but does not call it. Since ftrace.o should | 440 | * function but does not call it. Since ftrace.o should |
381 | * not be traced anyway, we just skip it. | 441 | * not be traced anyway, we just skip it. |
382 | */ | 442 | */ |
383 | len = strlen(argv[0]); | 443 | len = strlen(file); |
384 | if (len >= ftrace_size && | 444 | if (len >= ftrace_size && |
385 | strcmp(argv[0] + (len - ftrace_size), ftrace) == 0) | 445 | strcmp(file + (len - ftrace_size), ftrace) == 0) |
386 | continue; | 446 | continue; |
387 | 447 | ||
388 | switch (sjval) { | 448 | switch (sjval) { |
389 | default: { | 449 | default: |
390 | fprintf(stderr, "internal error: %s\n", argv[0]); | 450 | fprintf(stderr, "internal error: %s\n", file); |
391 | exit(1); | 451 | exit(1); |
392 | } break; | 452 | break; |
393 | case SJ_SETJMP: { /* normal sequence */ | 453 | case SJ_SETJMP: /* normal sequence */ |
394 | /* Avoid problems if early cleanup() */ | 454 | /* Avoid problems if early cleanup() */ |
395 | fd_map = -1; | 455 | fd_map = -1; |
396 | ehdr_curr = NULL; | 456 | ehdr_curr = NULL; |
397 | mmap_failed = 1; | 457 | mmap_failed = 1; |
398 | do_file(argv[0]); | 458 | do_file(file); |
399 | } break; | 459 | break; |
400 | case SJ_FAIL: { /* error in do_file or below */ | 460 | case SJ_FAIL: /* error in do_file or below */ |
401 | ++n_error; | 461 | ++n_error; |
402 | } break; | 462 | break; |
403 | case SJ_SUCCEED: { /* premature success */ | 463 | case SJ_SUCCEED: /* premature success */ |
404 | /* do nothing */ | 464 | /* do nothing */ |
405 | } break; | 465 | break; |
406 | } /* end switch */ | 466 | } /* end switch */ |
407 | } | 467 | } |
408 | return !!n_error; | 468 | return !!n_error; |
diff --git a/scripts/recordmcount.h b/scripts/recordmcount.h index baf187bee983..4be60364a405 100644 --- a/scripts/recordmcount.h +++ b/scripts/recordmcount.h | |||
@@ -22,11 +22,15 @@ | |||
22 | #undef is_fake_mcount | 22 | #undef is_fake_mcount |
23 | #undef fn_is_fake_mcount | 23 | #undef fn_is_fake_mcount |
24 | #undef MIPS_is_fake_mcount | 24 | #undef MIPS_is_fake_mcount |
25 | #undef mcount_adjust | ||
25 | #undef sift_rel_mcount | 26 | #undef sift_rel_mcount |
27 | #undef nop_mcount | ||
26 | #undef find_secsym_ndx | 28 | #undef find_secsym_ndx |
27 | #undef __has_rel_mcount | 29 | #undef __has_rel_mcount |
28 | #undef has_rel_mcount | 30 | #undef has_rel_mcount |
29 | #undef tot_relsize | 31 | #undef tot_relsize |
32 | #undef get_mcountsym | ||
33 | #undef get_sym_str_and_relp | ||
30 | #undef do_func | 34 | #undef do_func |
31 | #undef Elf_Addr | 35 | #undef Elf_Addr |
32 | #undef Elf_Ehdr | 36 | #undef Elf_Ehdr |
@@ -49,14 +53,18 @@ | |||
49 | #ifdef RECORD_MCOUNT_64 | 53 | #ifdef RECORD_MCOUNT_64 |
50 | # define append_func append64 | 54 | # define append_func append64 |
51 | # define sift_rel_mcount sift64_rel_mcount | 55 | # define sift_rel_mcount sift64_rel_mcount |
56 | # define nop_mcount nop_mcount_64 | ||
52 | # define find_secsym_ndx find64_secsym_ndx | 57 | # define find_secsym_ndx find64_secsym_ndx |
53 | # define __has_rel_mcount __has64_rel_mcount | 58 | # define __has_rel_mcount __has64_rel_mcount |
54 | # define has_rel_mcount has64_rel_mcount | 59 | # define has_rel_mcount has64_rel_mcount |
55 | # define tot_relsize tot64_relsize | 60 | # define tot_relsize tot64_relsize |
61 | # define get_sym_str_and_relp get_sym_str_and_relp_64 | ||
56 | # define do_func do64 | 62 | # define do_func do64 |
63 | # define get_mcountsym get_mcountsym_64 | ||
57 | # define is_fake_mcount is_fake_mcount64 | 64 | # define is_fake_mcount is_fake_mcount64 |
58 | # define fn_is_fake_mcount fn_is_fake_mcount64 | 65 | # define fn_is_fake_mcount fn_is_fake_mcount64 |
59 | # define MIPS_is_fake_mcount MIPS64_is_fake_mcount | 66 | # define MIPS_is_fake_mcount MIPS64_is_fake_mcount |
67 | # define mcount_adjust mcount_adjust_64 | ||
60 | # define Elf_Addr Elf64_Addr | 68 | # define Elf_Addr Elf64_Addr |
61 | # define Elf_Ehdr Elf64_Ehdr | 69 | # define Elf_Ehdr Elf64_Ehdr |
62 | # define Elf_Shdr Elf64_Shdr | 70 | # define Elf_Shdr Elf64_Shdr |
@@ -77,14 +85,18 @@ | |||
77 | #else | 85 | #else |
78 | # define append_func append32 | 86 | # define append_func append32 |
79 | # define sift_rel_mcount sift32_rel_mcount | 87 | # define sift_rel_mcount sift32_rel_mcount |
88 | # define nop_mcount nop_mcount_32 | ||
80 | # define find_secsym_ndx find32_secsym_ndx | 89 | # define find_secsym_ndx find32_secsym_ndx |
81 | # define __has_rel_mcount __has32_rel_mcount | 90 | # define __has_rel_mcount __has32_rel_mcount |
82 | # define has_rel_mcount has32_rel_mcount | 91 | # define has_rel_mcount has32_rel_mcount |
83 | # define tot_relsize tot32_relsize | 92 | # define tot_relsize tot32_relsize |
93 | # define get_sym_str_and_relp get_sym_str_and_relp_32 | ||
84 | # define do_func do32 | 94 | # define do_func do32 |
95 | # define get_mcountsym get_mcountsym_32 | ||
85 | # define is_fake_mcount is_fake_mcount32 | 96 | # define is_fake_mcount is_fake_mcount32 |
86 | # define fn_is_fake_mcount fn_is_fake_mcount32 | 97 | # define fn_is_fake_mcount fn_is_fake_mcount32 |
87 | # define MIPS_is_fake_mcount MIPS32_is_fake_mcount | 98 | # define MIPS_is_fake_mcount MIPS32_is_fake_mcount |
99 | # define mcount_adjust mcount_adjust_32 | ||
88 | # define Elf_Addr Elf32_Addr | 100 | # define Elf_Addr Elf32_Addr |
89 | # define Elf_Ehdr Elf32_Ehdr | 101 | # define Elf_Ehdr Elf32_Ehdr |
90 | # define Elf_Shdr Elf32_Shdr | 102 | # define Elf_Shdr Elf32_Shdr |
@@ -123,6 +135,8 @@ static void fn_ELF_R_INFO(Elf_Rel *const rp, unsigned sym, unsigned type) | |||
123 | } | 135 | } |
124 | static void (*Elf_r_info)(Elf_Rel *const rp, unsigned sym, unsigned type) = fn_ELF_R_INFO; | 136 | static void (*Elf_r_info)(Elf_Rel *const rp, unsigned sym, unsigned type) = fn_ELF_R_INFO; |
125 | 137 | ||
138 | static int mcount_adjust = 0; | ||
139 | |||
126 | /* | 140 | /* |
127 | * MIPS mcount long call has 2 _mcount symbols, only the position of the 1st | 141 | * MIPS mcount long call has 2 _mcount symbols, only the position of the 1st |
128 | * _mcount symbol is needed for dynamic function tracer, with it, to disable | 142 | * _mcount symbol is needed for dynamic function tracer, with it, to disable |
@@ -234,6 +248,49 @@ static void append_func(Elf_Ehdr *const ehdr, | |||
234 | uwrite(fd_map, ehdr, sizeof(*ehdr)); | 248 | uwrite(fd_map, ehdr, sizeof(*ehdr)); |
235 | } | 249 | } |
236 | 250 | ||
251 | static unsigned get_mcountsym(Elf_Sym const *const sym0, | ||
252 | Elf_Rel const *relp, | ||
253 | char const *const str0) | ||
254 | { | ||
255 | unsigned mcountsym = 0; | ||
256 | |||
257 | Elf_Sym const *const symp = | ||
258 | &sym0[Elf_r_sym(relp)]; | ||
259 | char const *symname = &str0[w(symp->st_name)]; | ||
260 | char const *mcount = gpfx == '_' ? "_mcount" : "mcount"; | ||
261 | |||
262 | if (symname[0] == '.') | ||
263 | ++symname; /* ppc64 hack */ | ||
264 | if (strcmp(mcount, symname) == 0 || | ||
265 | (altmcount && strcmp(altmcount, symname) == 0)) | ||
266 | mcountsym = Elf_r_sym(relp); | ||
267 | |||
268 | return mcountsym; | ||
269 | } | ||
270 | |||
271 | static void get_sym_str_and_relp(Elf_Shdr const *const relhdr, | ||
272 | Elf_Ehdr const *const ehdr, | ||
273 | Elf_Sym const **sym0, | ||
274 | char const **str0, | ||
275 | Elf_Rel const **relp) | ||
276 | { | ||
277 | Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff) | ||
278 | + (void *)ehdr); | ||
279 | unsigned const symsec_sh_link = w(relhdr->sh_link); | ||
280 | Elf_Shdr const *const symsec = &shdr0[symsec_sh_link]; | ||
281 | Elf_Shdr const *const strsec = &shdr0[w(symsec->sh_link)]; | ||
282 | Elf_Rel const *const rel0 = (Elf_Rel const *)(_w(relhdr->sh_offset) | ||
283 | + (void *)ehdr); | ||
284 | |||
285 | *sym0 = (Elf_Sym const *)(_w(symsec->sh_offset) | ||
286 | + (void *)ehdr); | ||
287 | |||
288 | *str0 = (char const *)(_w(strsec->sh_offset) | ||
289 | + (void *)ehdr); | ||
290 | |||
291 | *relp = rel0; | ||
292 | } | ||
293 | |||
237 | /* | 294 | /* |
238 | * Look at the relocations in order to find the calls to mcount. | 295 | * Look at the relocations in order to find the calls to mcount. |
239 | * Accumulate the section offsets that are found, and their relocation info, | 296 | * Accumulate the section offsets that are found, and their relocation info, |
@@ -250,47 +307,27 @@ static uint_t *sift_rel_mcount(uint_t *mlocp, | |||
250 | { | 307 | { |
251 | uint_t *const mloc0 = mlocp; | 308 | uint_t *const mloc0 = mlocp; |
252 | Elf_Rel *mrelp = *mrelpp; | 309 | Elf_Rel *mrelp = *mrelpp; |
253 | Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff) | 310 | Elf_Sym const *sym0; |
254 | + (void *)ehdr); | 311 | char const *str0; |
255 | unsigned const symsec_sh_link = w(relhdr->sh_link); | 312 | Elf_Rel const *relp; |
256 | Elf_Shdr const *const symsec = &shdr0[symsec_sh_link]; | ||
257 | Elf_Sym const *const sym0 = (Elf_Sym const *)(_w(symsec->sh_offset) | ||
258 | + (void *)ehdr); | ||
259 | |||
260 | Elf_Shdr const *const strsec = &shdr0[w(symsec->sh_link)]; | ||
261 | char const *const str0 = (char const *)(_w(strsec->sh_offset) | ||
262 | + (void *)ehdr); | ||
263 | |||
264 | Elf_Rel const *const rel0 = (Elf_Rel const *)(_w(relhdr->sh_offset) | ||
265 | + (void *)ehdr); | ||
266 | unsigned rel_entsize = _w(relhdr->sh_entsize); | 313 | unsigned rel_entsize = _w(relhdr->sh_entsize); |
267 | unsigned const nrel = _w(relhdr->sh_size) / rel_entsize; | 314 | unsigned const nrel = _w(relhdr->sh_size) / rel_entsize; |
268 | Elf_Rel const *relp = rel0; | ||
269 | |||
270 | unsigned mcountsym = 0; | 315 | unsigned mcountsym = 0; |
271 | unsigned t; | 316 | unsigned t; |
272 | 317 | ||
318 | get_sym_str_and_relp(relhdr, ehdr, &sym0, &str0, &relp); | ||
319 | |||
273 | for (t = nrel; t; --t) { | 320 | for (t = nrel; t; --t) { |
274 | if (!mcountsym) { | 321 | if (!mcountsym) |
275 | Elf_Sym const *const symp = | 322 | mcountsym = get_mcountsym(sym0, relp, str0); |
276 | &sym0[Elf_r_sym(relp)]; | ||
277 | char const *symname = &str0[w(symp->st_name)]; | ||
278 | char const *mcount = '_' == gpfx ? "_mcount" : "mcount"; | ||
279 | |||
280 | if ('.' == symname[0]) | ||
281 | ++symname; /* ppc64 hack */ | ||
282 | if (0 == strcmp(mcount, symname) || | ||
283 | (altmcount && 0 == strcmp(altmcount, symname))) | ||
284 | mcountsym = Elf_r_sym(relp); | ||
285 | } | ||
286 | 323 | ||
287 | if (mcountsym == Elf_r_sym(relp) && !is_fake_mcount(relp)) { | 324 | if (mcountsym == Elf_r_sym(relp) && !is_fake_mcount(relp)) { |
288 | uint_t const addend = _w(_w(relp->r_offset) - recval); | 325 | uint_t const addend = |
289 | 326 | _w(_w(relp->r_offset) - recval + mcount_adjust); | |
290 | mrelp->r_offset = _w(offbase | 327 | mrelp->r_offset = _w(offbase |
291 | + ((void *)mlocp - (void *)mloc0)); | 328 | + ((void *)mlocp - (void *)mloc0)); |
292 | Elf_r_info(mrelp, recsym, reltype); | 329 | Elf_r_info(mrelp, recsym, reltype); |
293 | if (sizeof(Elf_Rela) == rel_entsize) { | 330 | if (rel_entsize == sizeof(Elf_Rela)) { |
294 | ((Elf_Rela *)mrelp)->r_addend = addend; | 331 | ((Elf_Rela *)mrelp)->r_addend = addend; |
295 | *mlocp++ = 0; | 332 | *mlocp++ = 0; |
296 | } else | 333 | } else |
@@ -304,6 +341,63 @@ static uint_t *sift_rel_mcount(uint_t *mlocp, | |||
304 | return mlocp; | 341 | return mlocp; |
305 | } | 342 | } |
306 | 343 | ||
344 | /* | ||
345 | * Read the relocation table again, but this time its called on sections | ||
346 | * that are not going to be traced. The mcount calls here will be converted | ||
347 | * into nops. | ||
348 | */ | ||
349 | static void nop_mcount(Elf_Shdr const *const relhdr, | ||
350 | Elf_Ehdr const *const ehdr, | ||
351 | const char *const txtname) | ||
352 | { | ||
353 | Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff) | ||
354 | + (void *)ehdr); | ||
355 | Elf_Sym const *sym0; | ||
356 | char const *str0; | ||
357 | Elf_Rel const *relp; | ||
358 | Elf_Shdr const *const shdr = &shdr0[w(relhdr->sh_info)]; | ||
359 | unsigned rel_entsize = _w(relhdr->sh_entsize); | ||
360 | unsigned const nrel = _w(relhdr->sh_size) / rel_entsize; | ||
361 | unsigned mcountsym = 0; | ||
362 | unsigned t; | ||
363 | int once = 0; | ||
364 | |||
365 | get_sym_str_and_relp(relhdr, ehdr, &sym0, &str0, &relp); | ||
366 | |||
367 | for (t = nrel; t; --t) { | ||
368 | int ret = -1; | ||
369 | |||
370 | if (!mcountsym) | ||
371 | mcountsym = get_mcountsym(sym0, relp, str0); | ||
372 | |||
373 | if (mcountsym == Elf_r_sym(relp) && !is_fake_mcount(relp)) { | ||
374 | if (make_nop) | ||
375 | ret = make_nop((void *)ehdr, shdr->sh_offset + relp->r_offset); | ||
376 | if (warn_on_notrace_sect && !once) { | ||
377 | printf("Section %s has mcount callers being ignored\n", | ||
378 | txtname); | ||
379 | once = 1; | ||
380 | /* just warn? */ | ||
381 | if (!make_nop) | ||
382 | return; | ||
383 | } | ||
384 | } | ||
385 | |||
386 | /* | ||
387 | * If we successfully removed the mcount, mark the relocation | ||
388 | * as a nop (don't do anything with it). | ||
389 | */ | ||
390 | if (!ret) { | ||
391 | Elf_Rel rel; | ||
392 | rel = *(Elf_Rel *)relp; | ||
393 | Elf_r_info(&rel, Elf_r_sym(relp), rel_type_nop); | ||
394 | ulseek(fd_map, (void *)relp - (void *)ehdr, SEEK_SET); | ||
395 | uwrite(fd_map, &rel, sizeof(rel)); | ||
396 | } | ||
397 | relp = (Elf_Rel const *)(rel_entsize + (void *)relp); | ||
398 | } | ||
399 | } | ||
400 | |||
307 | 401 | ||
308 | /* | 402 | /* |
309 | * Find a symbol in the given section, to be used as the base for relocating | 403 | * Find a symbol in the given section, to be used as the base for relocating |
@@ -354,13 +448,13 @@ __has_rel_mcount(Elf_Shdr const *const relhdr, /* is SHT_REL or SHT_RELA */ | |||
354 | Elf_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)]; | 448 | Elf_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)]; |
355 | char const *const txtname = &shstrtab[w(txthdr->sh_name)]; | 449 | char const *const txtname = &shstrtab[w(txthdr->sh_name)]; |
356 | 450 | ||
357 | if (0 == strcmp("__mcount_loc", txtname)) { | 451 | if (strcmp("__mcount_loc", txtname) == 0) { |
358 | fprintf(stderr, "warning: __mcount_loc already exists: %s\n", | 452 | fprintf(stderr, "warning: __mcount_loc already exists: %s\n", |
359 | fname); | 453 | fname); |
360 | succeed_file(); | 454 | succeed_file(); |
361 | } | 455 | } |
362 | if (SHT_PROGBITS != w(txthdr->sh_type) || | 456 | if (w(txthdr->sh_type) != SHT_PROGBITS || |
363 | !is_mcounted_section_name(txtname)) | 457 | !(w(txthdr->sh_flags) & SHF_EXECINSTR)) |
364 | return NULL; | 458 | return NULL; |
365 | return txtname; | 459 | return txtname; |
366 | } | 460 | } |
@@ -370,7 +464,7 @@ static char const *has_rel_mcount(Elf_Shdr const *const relhdr, | |||
370 | char const *const shstrtab, | 464 | char const *const shstrtab, |
371 | char const *const fname) | 465 | char const *const fname) |
372 | { | 466 | { |
373 | if (SHT_REL != w(relhdr->sh_type) && SHT_RELA != w(relhdr->sh_type)) | 467 | if (w(relhdr->sh_type) != SHT_REL && w(relhdr->sh_type) != SHT_RELA) |
374 | return NULL; | 468 | return NULL; |
375 | return __has_rel_mcount(relhdr, shdr0, shstrtab, fname); | 469 | return __has_rel_mcount(relhdr, shdr0, shstrtab, fname); |
376 | } | 470 | } |
@@ -383,9 +477,11 @@ static unsigned tot_relsize(Elf_Shdr const *const shdr0, | |||
383 | { | 477 | { |
384 | unsigned totrelsz = 0; | 478 | unsigned totrelsz = 0; |
385 | Elf_Shdr const *shdrp = shdr0; | 479 | Elf_Shdr const *shdrp = shdr0; |
480 | char const *txtname; | ||
386 | 481 | ||
387 | for (; nhdr; --nhdr, ++shdrp) { | 482 | for (; nhdr; --nhdr, ++shdrp) { |
388 | if (has_rel_mcount(shdrp, shdr0, shstrtab, fname)) | 483 | txtname = has_rel_mcount(shdrp, shdr0, shstrtab, fname); |
484 | if (txtname && is_mcounted_section_name(txtname)) | ||
389 | totrelsz += _w(shdrp->sh_size); | 485 | totrelsz += _w(shdrp->sh_size); |
390 | } | 486 | } |
391 | return totrelsz; | 487 | return totrelsz; |
@@ -421,7 +517,7 @@ do_func(Elf_Ehdr *const ehdr, char const *const fname, unsigned const reltype) | |||
421 | for (relhdr = shdr0, k = nhdr; k; --k, ++relhdr) { | 517 | for (relhdr = shdr0, k = nhdr; k; --k, ++relhdr) { |
422 | char const *const txtname = has_rel_mcount(relhdr, shdr0, | 518 | char const *const txtname = has_rel_mcount(relhdr, shdr0, |
423 | shstrtab, fname); | 519 | shstrtab, fname); |
424 | if (txtname) { | 520 | if (txtname && is_mcounted_section_name(txtname)) { |
425 | uint_t recval = 0; | 521 | uint_t recval = 0; |
426 | unsigned const recsym = find_secsym_ndx( | 522 | unsigned const recsym = find_secsym_ndx( |
427 | w(relhdr->sh_info), txtname, &recval, | 523 | w(relhdr->sh_info), txtname, &recval, |
@@ -432,6 +528,12 @@ do_func(Elf_Ehdr *const ehdr, char const *const fname, unsigned const reltype) | |||
432 | mlocp = sift_rel_mcount(mlocp, | 528 | mlocp = sift_rel_mcount(mlocp, |
433 | (void *)mlocp - (void *)mloc0, &mrelp, | 529 | (void *)mlocp - (void *)mloc0, &mrelp, |
434 | relhdr, ehdr, recsym, recval, reltype); | 530 | relhdr, ehdr, recsym, recval, reltype); |
531 | } else if (txtname && (warn_on_notrace_sect || make_nop)) { | ||
532 | /* | ||
533 | * This section is ignored by ftrace, but still | ||
534 | * has mcount calls. Convert them to nops now. | ||
535 | */ | ||
536 | nop_mcount(relhdr, ehdr, txtname); | ||
435 | } | 537 | } |
436 | } | 538 | } |
437 | if (mloc0 != mlocp) { | 539 | if (mloc0 != mlocp) { |
diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index 4be0deea71ca..858966ab019c 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl | |||
@@ -134,6 +134,7 @@ my %text_sections = ( | |||
134 | ".sched.text" => 1, | 134 | ".sched.text" => 1, |
135 | ".spinlock.text" => 1, | 135 | ".spinlock.text" => 1, |
136 | ".irqentry.text" => 1, | 136 | ".irqentry.text" => 1, |
137 | ".kprobes.text" => 1, | ||
137 | ".text.unlikely" => 1, | 138 | ".text.unlikely" => 1, |
138 | ); | 139 | ); |
139 | 140 | ||
@@ -222,6 +223,7 @@ if ($arch eq "x86_64") { | |||
222 | $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\smcount([+-]0x[0-9a-zA-Z]+)?\$"; | 223 | $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\smcount([+-]0x[0-9a-zA-Z]+)?\$"; |
223 | $type = ".quad"; | 224 | $type = ".quad"; |
224 | $alignment = 8; | 225 | $alignment = 8; |
226 | $mcount_adjust = -1; | ||
225 | 227 | ||
226 | # force flags for this arch | 228 | # force flags for this arch |
227 | $ld .= " -m elf_x86_64"; | 229 | $ld .= " -m elf_x86_64"; |
@@ -231,6 +233,7 @@ if ($arch eq "x86_64") { | |||
231 | 233 | ||
232 | } elsif ($arch eq "i386") { | 234 | } elsif ($arch eq "i386") { |
233 | $alignment = 4; | 235 | $alignment = 4; |
236 | $mcount_adjust = -1; | ||
234 | 237 | ||
235 | # force flags for this arch | 238 | # force flags for this arch |
236 | $ld .= " -m elf_i386"; | 239 | $ld .= " -m elf_i386"; |
@@ -240,12 +243,14 @@ if ($arch eq "x86_64") { | |||
240 | 243 | ||
241 | } elsif ($arch eq "s390" && $bits == 32) { | 244 | } elsif ($arch eq "s390" && $bits == 32) { |
242 | $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_32\\s+_mcount\$"; | 245 | $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_32\\s+_mcount\$"; |
246 | $mcount_adjust = -4; | ||
243 | $alignment = 4; | 247 | $alignment = 4; |
244 | $ld .= " -m elf_s390"; | 248 | $ld .= " -m elf_s390"; |
245 | $cc .= " -m31"; | 249 | $cc .= " -m31"; |
246 | 250 | ||
247 | } elsif ($arch eq "s390" && $bits == 64) { | 251 | } elsif ($arch eq "s390" && $bits == 64) { |
248 | $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_(PC|PLT)32DBL\\s+_mcount\\+0x2\$"; | 252 | $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_(PC|PLT)32DBL\\s+_mcount\\+0x2\$"; |
253 | $mcount_adjust = -8; | ||
249 | $alignment = 8; | 254 | $alignment = 8; |
250 | $type = ".quad"; | 255 | $type = ".quad"; |
251 | $ld .= " -m elf64_s390"; | 256 | $ld .= " -m elf64_s390"; |
diff --git a/tools/perf/Documentation/perf-script.txt b/tools/perf/Documentation/perf-script.txt index 66f040b30729..86c87e214b11 100644 --- a/tools/perf/Documentation/perf-script.txt +++ b/tools/perf/Documentation/perf-script.txt | |||
@@ -113,13 +113,61 @@ OPTIONS | |||
113 | Do various checks like samples ordering and lost events. | 113 | Do various checks like samples ordering and lost events. |
114 | 114 | ||
115 | -f:: | 115 | -f:: |
116 | --fields | 116 | --fields:: |
117 | Comma separated list of fields to print. Options are: | 117 | Comma separated list of fields to print. Options are: |
118 | comm, tid, pid, time, cpu, event, trace, sym. Field | 118 | comm, tid, pid, time, cpu, event, trace, sym. Field |
119 | list must be prepended with the type, trace, sw or hw, | 119 | list can be prepended with the type, trace, sw or hw, |
120 | to indicate to which event type the field list applies. | 120 | to indicate to which event type the field list applies. |
121 | e.g., -f sw:comm,tid,time,sym and -f trace:time,cpu,trace | 121 | e.g., -f sw:comm,tid,time,sym and -f trace:time,cpu,trace |
122 | 122 | ||
123 | perf script -f <fields> | ||
124 | |||
125 | is equivalent to: | ||
126 | |||
127 | perf script -f trace:<fields> -f sw:<fields> -f hw:<fields> | ||
128 | |||
129 | i.e., the specified fields apply to all event types if the type string | ||
130 | is not given. | ||
131 | |||
132 | The arguments are processed in the order received. A later usage can | ||
133 | reset a prior request. e.g.: | ||
134 | |||
135 | -f trace: -f comm,tid,time,sym | ||
136 | |||
137 | The first -f suppresses trace events (field list is ""), but then the | ||
138 | second invocation sets the fields to comm,tid,time,sym. In this case a | ||
139 | warning is given to the user: | ||
140 | |||
141 | "Overriding previous field request for all events." | ||
142 | |||
143 | Alternativey, consider the order: | ||
144 | |||
145 | -f comm,tid,time,sym -f trace: | ||
146 | |||
147 | The first -f sets the fields for all events and the second -f | ||
148 | suppresses trace events. The user is given a warning message about | ||
149 | the override, and the result of the above is that only S/W and H/W | ||
150 | events are displayed with the given fields. | ||
151 | |||
152 | For the 'wildcard' option if a user selected field is invalid for an | ||
153 | event type, a message is displayed to the user that the option is | ||
154 | ignored for that type. For example: | ||
155 | |||
156 | $ perf script -f comm,tid,trace | ||
157 | 'trace' not valid for hardware events. Ignoring. | ||
158 | 'trace' not valid for software events. Ignoring. | ||
159 | |||
160 | Alternatively, if the type is given an invalid field is specified it | ||
161 | is an error. For example: | ||
162 | |||
163 | perf script -v -f sw:comm,tid,trace | ||
164 | 'trace' not valid for software events. | ||
165 | |||
166 | At this point usage is displayed, and perf-script exits. | ||
167 | |||
168 | Finally, a user may not set fields to none for all event types. | ||
169 | i.e., -f "" is not allowed. | ||
170 | |||
123 | -k:: | 171 | -k:: |
124 | --vmlinux=<file>:: | 172 | --vmlinux=<file>:: |
125 | vmlinux pathname | 173 | vmlinux pathname |
diff --git a/tools/perf/Makefile b/tools/perf/Makefile index 0c542563ea6c..1455413ec7a7 100644 --- a/tools/perf/Makefile +++ b/tools/perf/Makefile | |||
@@ -5,6 +5,8 @@ endif | |||
5 | # The default target of this Makefile is... | 5 | # The default target of this Makefile is... |
6 | all: | 6 | all: |
7 | 7 | ||
8 | include config/utilities.mak | ||
9 | |||
8 | ifneq ($(OUTPUT),) | 10 | ifneq ($(OUTPUT),) |
9 | # check that the output directory actually exists | 11 | # check that the output directory actually exists |
10 | OUTDIR := $(shell cd $(OUTPUT) && /bin/pwd) | 12 | OUTDIR := $(shell cd $(OUTPUT) && /bin/pwd) |
@@ -13,6 +15,12 @@ endif | |||
13 | 15 | ||
14 | # Define V to have a more verbose compile. | 16 | # Define V to have a more verbose compile. |
15 | # | 17 | # |
18 | # Define PYTHON to point to the python binary if the default | ||
19 | # `python' is not correct; for example: PYTHON=python2 | ||
20 | # | ||
21 | # Define PYTHON_CONFIG to point to the python-config binary if | ||
22 | # the default `$(PYTHON)-config' is not correct. | ||
23 | # | ||
16 | # Define ASCIIDOC8 if you want to format documentation with AsciiDoc 8 | 24 | # Define ASCIIDOC8 if you want to format documentation with AsciiDoc 8 |
17 | # | 25 | # |
18 | # Define DOCBOOK_XSL_172 if you want to format man pages with DocBook XSL v1.72. | 26 | # Define DOCBOOK_XSL_172 if you want to format man pages with DocBook XSL v1.72. |
@@ -134,7 +142,7 @@ INSTALL = install | |||
134 | # explicitly what architecture to check for. Fix this up for yours.. | 142 | # explicitly what architecture to check for. Fix this up for yours.. |
135 | SPARSE_FLAGS = -D__BIG_ENDIAN__ -D__powerpc__ | 143 | SPARSE_FLAGS = -D__BIG_ENDIAN__ -D__powerpc__ |
136 | 144 | ||
137 | -include feature-tests.mak | 145 | -include config/feature-tests.mak |
138 | 146 | ||
139 | ifeq ($(call try-cc,$(SOURCE_HELLO),-Werror -fstack-protector-all),y) | 147 | ifeq ($(call try-cc,$(SOURCE_HELLO),-Werror -fstack-protector-all),y) |
140 | CFLAGS := $(CFLAGS) -fstack-protector-all | 148 | CFLAGS := $(CFLAGS) -fstack-protector-all |
@@ -169,12 +177,10 @@ grep-libs = $(filter -l%,$(1)) | |||
169 | strip-libs = $(filter-out -l%,$(1)) | 177 | strip-libs = $(filter-out -l%,$(1)) |
170 | 178 | ||
171 | $(OUTPUT)python/perf.so: $(PYRF_OBJS) | 179 | $(OUTPUT)python/perf.so: $(PYRF_OBJS) |
172 | $(QUIET_GEN)( \ | 180 | $(QUIET_GEN)CFLAGS='$(BASIC_CFLAGS)' $(PYTHON_WORD) util/setup.py \ |
173 | export CFLAGS="$(BASIC_CFLAGS)"; \ | 181 | --quiet build_ext \ |
174 | python util/setup.py --quiet build_ext --build-lib='$(OUTPUT)python' \ | 182 | --build-lib='$(OUTPUT)python' \ |
175 | --build-temp='$(OUTPUT)python/temp' \ | 183 | --build-temp='$(OUTPUT)python/temp' |
176 | ) | ||
177 | |||
178 | # | 184 | # |
179 | # No Perl scripts right now: | 185 | # No Perl scripts right now: |
180 | # | 186 | # |
@@ -479,24 +485,74 @@ else | |||
479 | endif | 485 | endif |
480 | endif | 486 | endif |
481 | 487 | ||
482 | ifdef NO_LIBPYTHON | 488 | disable-python = $(eval $(disable-python_code)) |
483 | BASIC_CFLAGS += -DNO_LIBPYTHON | 489 | define disable-python_code |
490 | BASIC_CFLAGS += -DNO_LIBPYTHON | ||
491 | $(if $(1),$(warning No $(1) was found)) | ||
492 | $(warning Python support won't be built) | ||
493 | endef | ||
494 | |||
495 | override PYTHON := \ | ||
496 | $(call get-executable-or-default,PYTHON,python) | ||
497 | |||
498 | ifndef PYTHON | ||
499 | $(call disable-python,python interpreter) | ||
500 | python-clean := | ||
484 | else | 501 | else |
485 | PYTHON_EMBED_LDOPTS = $(shell python-config --ldflags 2>/dev/null) | 502 | |
486 | PYTHON_EMBED_LDFLAGS = $(call strip-libs,$(PYTHON_EMBED_LDOPTS)) | 503 | PYTHON_WORD := $(call shell-wordify,$(PYTHON)) |
487 | PYTHON_EMBED_LIBADD = $(call grep-libs,$(PYTHON_EMBED_LDOPTS)) | 504 | |
488 | PYTHON_EMBED_CCOPTS = `python-config --cflags 2>/dev/null` | 505 | python-clean := $(PYTHON_WORD) util/setup.py clean \ |
489 | FLAGS_PYTHON_EMBED=$(PYTHON_EMBED_CCOPTS) $(PYTHON_EMBED_LDOPTS) | 506 | --build-lib='$(OUTPUT)python' \ |
490 | ifneq ($(call try-cc,$(SOURCE_PYTHON_EMBED),$(FLAGS_PYTHON_EMBED)),y) | 507 | --build-temp='$(OUTPUT)python/temp' |
491 | msg := $(warning No Python.h found, install python-dev[el] to have python support in 'perf script' and to build the python bindings) | 508 | |
492 | BASIC_CFLAGS += -DNO_LIBPYTHON | 509 | ifdef NO_LIBPYTHON |
493 | else | 510 | $(call disable-python) |
494 | ALL_LDFLAGS += $(PYTHON_EMBED_LDFLAGS) | 511 | else |
495 | EXTLIBS += $(PYTHON_EMBED_LIBADD) | 512 | |
496 | LIB_OBJS += $(OUTPUT)util/scripting-engines/trace-event-python.o | 513 | override PYTHON_CONFIG := \ |
497 | LIB_OBJS += $(OUTPUT)scripts/python/Perf-Trace-Util/Context.o | 514 | $(call get-executable-or-default,PYTHON_CONFIG,$(PYTHON)-config) |
498 | LANG_BINDINGS += $(OUTPUT)python/perf.so | 515 | |
499 | endif | 516 | ifndef PYTHON_CONFIG |
517 | $(call disable-python,python-config tool) | ||
518 | else | ||
519 | |||
520 | PYTHON_CONFIG_SQ := $(call shell-sq,$(PYTHON_CONFIG)) | ||
521 | |||
522 | PYTHON_EMBED_LDOPTS := $(shell $(PYTHON_CONFIG_SQ) --ldflags 2>/dev/null) | ||
523 | PYTHON_EMBED_LDFLAGS := $(call strip-libs,$(PYTHON_EMBED_LDOPTS)) | ||
524 | PYTHON_EMBED_LIBADD := $(call grep-libs,$(PYTHON_EMBED_LDOPTS)) | ||
525 | PYTHON_EMBED_CCOPTS := $(shell $(PYTHON_CONFIG_SQ) --cflags 2>/dev/null) | ||
526 | FLAGS_PYTHON_EMBED := $(PYTHON_EMBED_CCOPTS) $(PYTHON_EMBED_LDOPTS) | ||
527 | |||
528 | ifneq ($(call try-cc,$(SOURCE_PYTHON_EMBED),$(FLAGS_PYTHON_EMBED)),y) | ||
529 | $(call disable-python,Python.h (for Python 2.x)) | ||
530 | else | ||
531 | |||
532 | ifneq ($(call try-cc,$(SOURCE_PYTHON_VERSION),$(FLAGS_PYTHON_EMBED)),y) | ||
533 | $(warning Python 3 is not yet supported; please set) | ||
534 | $(warning PYTHON and/or PYTHON_CONFIG appropriately.) | ||
535 | $(warning If you also have Python 2 installed, then) | ||
536 | $(warning try something like:) | ||
537 | $(warning $(and ,)) | ||
538 | $(warning $(and ,) make PYTHON=python2) | ||
539 | $(warning $(and ,)) | ||
540 | $(warning Otherwise, disable Python support entirely:) | ||
541 | $(warning $(and ,)) | ||
542 | $(warning $(and ,) make NO_LIBPYTHON=1) | ||
543 | $(warning $(and ,)) | ||
544 | $(error $(and ,)) | ||
545 | else | ||
546 | ALL_LDFLAGS += $(PYTHON_EMBED_LDFLAGS) | ||
547 | EXTLIBS += $(PYTHON_EMBED_LIBADD) | ||
548 | LIB_OBJS += $(OUTPUT)util/scripting-engines/trace-event-python.o | ||
549 | LIB_OBJS += $(OUTPUT)scripts/python/Perf-Trace-Util/Context.o | ||
550 | LANG_BINDINGS += $(OUTPUT)python/perf.so | ||
551 | endif | ||
552 | |||
553 | endif | ||
554 | endif | ||
555 | endif | ||
500 | endif | 556 | endif |
501 | 557 | ||
502 | ifdef NO_DEMANGLE | 558 | ifdef NO_DEMANGLE |
@@ -837,8 +893,7 @@ clean: | |||
837 | $(RM) *.spec *.pyc *.pyo */*.pyc */*.pyo $(OUTPUT)common-cmds.h TAGS tags cscope* | 893 | $(RM) *.spec *.pyc *.pyo */*.pyc */*.pyo $(OUTPUT)common-cmds.h TAGS tags cscope* |
838 | $(MAKE) -C Documentation/ clean | 894 | $(MAKE) -C Documentation/ clean |
839 | $(RM) $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)PERF-CFLAGS | 895 | $(RM) $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)PERF-CFLAGS |
840 | @python util/setup.py clean --build-lib='$(OUTPUT)python' \ | 896 | $(python-clean) |
841 | --build-temp='$(OUTPUT)python/temp' | ||
842 | 897 | ||
843 | .PHONY: all install clean strip | 898 | .PHONY: all install clean strip |
844 | .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell | 899 | .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell |
diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c index ac574ea23917..974f6d3f4e53 100644 --- a/tools/perf/builtin-script.c +++ b/tools/perf/builtin-script.c | |||
@@ -49,57 +49,169 @@ struct output_option { | |||
49 | }; | 49 | }; |
50 | 50 | ||
51 | /* default set to maintain compatibility with current format */ | 51 | /* default set to maintain compatibility with current format */ |
52 | static u64 output_fields[PERF_TYPE_MAX] = { | 52 | static struct { |
53 | [PERF_TYPE_HARDWARE] = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | \ | 53 | bool user_set; |
54 | PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | \ | 54 | bool wildcard_set; |
55 | PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM, | 55 | u64 fields; |
56 | 56 | u64 invalid_fields; | |
57 | [PERF_TYPE_SOFTWARE] = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | \ | 57 | } output[PERF_TYPE_MAX] = { |
58 | PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | \ | 58 | |
59 | PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM, | 59 | [PERF_TYPE_HARDWARE] = { |
60 | 60 | .user_set = false, | |
61 | [PERF_TYPE_TRACEPOINT] = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | \ | 61 | |
62 | PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | \ | 62 | .fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | |
63 | PERF_OUTPUT_EVNAME | PERF_OUTPUT_TRACE, | 63 | PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | |
64 | PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM, | ||
65 | |||
66 | .invalid_fields = PERF_OUTPUT_TRACE, | ||
67 | }, | ||
68 | |||
69 | [PERF_TYPE_SOFTWARE] = { | ||
70 | .user_set = false, | ||
71 | |||
72 | .fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | | ||
73 | PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | | ||
74 | PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM, | ||
75 | |||
76 | .invalid_fields = PERF_OUTPUT_TRACE, | ||
77 | }, | ||
78 | |||
79 | [PERF_TYPE_TRACEPOINT] = { | ||
80 | .user_set = false, | ||
81 | |||
82 | .fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | | ||
83 | PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | | ||
84 | PERF_OUTPUT_EVNAME | PERF_OUTPUT_TRACE, | ||
85 | }, | ||
86 | |||
87 | [PERF_TYPE_RAW] = { | ||
88 | .user_set = false, | ||
89 | |||
90 | .fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | | ||
91 | PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | | ||
92 | PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM, | ||
93 | |||
94 | .invalid_fields = PERF_OUTPUT_TRACE, | ||
95 | }, | ||
64 | }; | 96 | }; |
65 | 97 | ||
66 | static bool output_set_by_user; | 98 | static bool output_set_by_user(void) |
99 | { | ||
100 | int j; | ||
101 | for (j = 0; j < PERF_TYPE_MAX; ++j) { | ||
102 | if (output[j].user_set) | ||
103 | return true; | ||
104 | } | ||
105 | return false; | ||
106 | } | ||
107 | |||
108 | static const char *output_field2str(enum perf_output_field field) | ||
109 | { | ||
110 | int i, imax = ARRAY_SIZE(all_output_options); | ||
111 | const char *str = ""; | ||
112 | |||
113 | for (i = 0; i < imax; ++i) { | ||
114 | if (all_output_options[i].field == field) { | ||
115 | str = all_output_options[i].str; | ||
116 | break; | ||
117 | } | ||
118 | } | ||
119 | return str; | ||
120 | } | ||
67 | 121 | ||
68 | #define PRINT_FIELD(x) (output_fields[attr->type] & PERF_OUTPUT_##x) | 122 | #define PRINT_FIELD(x) (output[attr->type].fields & PERF_OUTPUT_##x) |
69 | 123 | ||
70 | static int perf_session__check_attr(struct perf_session *session, | 124 | static int perf_event_attr__check_stype(struct perf_event_attr *attr, |
71 | struct perf_event_attr *attr) | 125 | u64 sample_type, const char *sample_msg, |
126 | enum perf_output_field field) | ||
72 | { | 127 | { |
128 | int type = attr->type; | ||
129 | const char *evname; | ||
130 | |||
131 | if (attr->sample_type & sample_type) | ||
132 | return 0; | ||
133 | |||
134 | if (output[type].user_set) { | ||
135 | evname = __event_name(attr->type, attr->config); | ||
136 | pr_err("Samples for '%s' event do not have %s attribute set. " | ||
137 | "Cannot print '%s' field.\n", | ||
138 | evname, sample_msg, output_field2str(field)); | ||
139 | return -1; | ||
140 | } | ||
141 | |||
142 | /* user did not ask for it explicitly so remove from the default list */ | ||
143 | output[type].fields &= ~field; | ||
144 | evname = __event_name(attr->type, attr->config); | ||
145 | pr_debug("Samples for '%s' event do not have %s attribute set. " | ||
146 | "Skipping '%s' field.\n", | ||
147 | evname, sample_msg, output_field2str(field)); | ||
148 | |||
149 | return 0; | ||
150 | } | ||
151 | |||
152 | static int perf_evsel__check_attr(struct perf_evsel *evsel, | ||
153 | struct perf_session *session) | ||
154 | { | ||
155 | struct perf_event_attr *attr = &evsel->attr; | ||
156 | |||
73 | if (PRINT_FIELD(TRACE) && | 157 | if (PRINT_FIELD(TRACE) && |
74 | !perf_session__has_traces(session, "record -R")) | 158 | !perf_session__has_traces(session, "record -R")) |
75 | return -EINVAL; | 159 | return -EINVAL; |
76 | 160 | ||
77 | if (PRINT_FIELD(SYM)) { | 161 | if (PRINT_FIELD(SYM)) { |
78 | if (!(session->sample_type & PERF_SAMPLE_IP)) { | 162 | if (perf_event_attr__check_stype(attr, PERF_SAMPLE_IP, "IP", |
79 | pr_err("Samples do not contain IP data.\n"); | 163 | PERF_OUTPUT_SYM)) |
80 | return -EINVAL; | 164 | return -EINVAL; |
81 | } | 165 | |
82 | if (!no_callchain && | 166 | if (!no_callchain && |
83 | !(session->sample_type & PERF_SAMPLE_CALLCHAIN)) | 167 | !(attr->sample_type & PERF_SAMPLE_CALLCHAIN)) |
84 | symbol_conf.use_callchain = false; | 168 | symbol_conf.use_callchain = false; |
85 | } | 169 | } |
86 | 170 | ||
87 | if ((PRINT_FIELD(PID) || PRINT_FIELD(TID)) && | 171 | if ((PRINT_FIELD(PID) || PRINT_FIELD(TID)) && |
88 | !(session->sample_type & PERF_SAMPLE_TID)) { | 172 | perf_event_attr__check_stype(attr, PERF_SAMPLE_TID, "TID", |
89 | pr_err("Samples do not contain TID/PID data.\n"); | 173 | PERF_OUTPUT_TID|PERF_OUTPUT_PID)) |
90 | return -EINVAL; | 174 | return -EINVAL; |
91 | } | ||
92 | 175 | ||
93 | if (PRINT_FIELD(TIME) && | 176 | if (PRINT_FIELD(TIME) && |
94 | !(session->sample_type & PERF_SAMPLE_TIME)) { | 177 | perf_event_attr__check_stype(attr, PERF_SAMPLE_TIME, "TIME", |
95 | pr_err("Samples do not contain timestamps.\n"); | 178 | PERF_OUTPUT_TIME)) |
96 | return -EINVAL; | 179 | return -EINVAL; |
97 | } | ||
98 | 180 | ||
99 | if (PRINT_FIELD(CPU) && | 181 | if (PRINT_FIELD(CPU) && |
100 | !(session->sample_type & PERF_SAMPLE_CPU)) { | 182 | perf_event_attr__check_stype(attr, PERF_SAMPLE_CPU, "CPU", |
101 | pr_err("Samples do not contain cpu.\n"); | 183 | PERF_OUTPUT_CPU)) |
102 | return -EINVAL; | 184 | return -EINVAL; |
185 | |||
186 | return 0; | ||
187 | } | ||
188 | |||
189 | /* | ||
190 | * verify all user requested events exist and the samples | ||
191 | * have the expected data | ||
192 | */ | ||
193 | static int perf_session__check_output_opt(struct perf_session *session) | ||
194 | { | ||
195 | int j; | ||
196 | struct perf_evsel *evsel; | ||
197 | |||
198 | for (j = 0; j < PERF_TYPE_MAX; ++j) { | ||
199 | evsel = perf_session__find_first_evtype(session, j); | ||
200 | |||
201 | /* | ||
202 | * even if fields is set to 0 (ie., show nothing) event must | ||
203 | * exist if user explicitly includes it on the command line | ||
204 | */ | ||
205 | if (!evsel && output[j].user_set && !output[j].wildcard_set) { | ||
206 | pr_err("%s events do not exist. " | ||
207 | "Remove corresponding -f option to proceed.\n", | ||
208 | event_type(j)); | ||
209 | return -1; | ||
210 | } | ||
211 | |||
212 | if (evsel && output[j].fields && | ||
213 | perf_evsel__check_attr(evsel, session)) | ||
214 | return -1; | ||
103 | } | 215 | } |
104 | 216 | ||
105 | return 0; | 217 | return 0; |
@@ -168,10 +280,7 @@ static void process_event(union perf_event *event __unused, | |||
168 | { | 280 | { |
169 | struct perf_event_attr *attr = &evsel->attr; | 281 | struct perf_event_attr *attr = &evsel->attr; |
170 | 282 | ||
171 | if (output_fields[attr->type] == 0) | 283 | if (output[attr->type].fields == 0) |
172 | return; | ||
173 | |||
174 | if (perf_session__check_attr(session, attr) < 0) | ||
175 | return; | 284 | return; |
176 | 285 | ||
177 | print_sample_start(sample, thread, attr); | 286 | print_sample_start(sample, thread, attr); |
@@ -451,6 +560,7 @@ static int parse_output_fields(const struct option *opt __used, | |||
451 | { | 560 | { |
452 | char *tok; | 561 | char *tok; |
453 | int i, imax = sizeof(all_output_options) / sizeof(struct output_option); | 562 | int i, imax = sizeof(all_output_options) / sizeof(struct output_option); |
563 | int j; | ||
454 | int rc = 0; | 564 | int rc = 0; |
455 | char *str = strdup(arg); | 565 | char *str = strdup(arg); |
456 | int type = -1; | 566 | int type = -1; |
@@ -458,52 +568,99 @@ static int parse_output_fields(const struct option *opt __used, | |||
458 | if (!str) | 568 | if (!str) |
459 | return -ENOMEM; | 569 | return -ENOMEM; |
460 | 570 | ||
461 | tok = strtok(str, ":"); | 571 | /* first word can state for which event type the user is specifying |
462 | if (!tok) { | 572 | * the fields. If no type exists, the specified fields apply to all |
463 | fprintf(stderr, | 573 | * event types found in the file minus the invalid fields for a type. |
464 | "Invalid field string - not prepended with type."); | ||
465 | return -EINVAL; | ||
466 | } | ||
467 | |||
468 | /* first word should state which event type user | ||
469 | * is specifying the fields | ||
470 | */ | 574 | */ |
471 | if (!strcmp(tok, "hw")) | 575 | tok = strchr(str, ':'); |
472 | type = PERF_TYPE_HARDWARE; | 576 | if (tok) { |
473 | else if (!strcmp(tok, "sw")) | 577 | *tok = '\0'; |
474 | type = PERF_TYPE_SOFTWARE; | 578 | tok++; |
475 | else if (!strcmp(tok, "trace")) | 579 | if (!strcmp(str, "hw")) |
476 | type = PERF_TYPE_TRACEPOINT; | 580 | type = PERF_TYPE_HARDWARE; |
477 | else { | 581 | else if (!strcmp(str, "sw")) |
478 | fprintf(stderr, "Invalid event type in field string."); | 582 | type = PERF_TYPE_SOFTWARE; |
479 | return -EINVAL; | 583 | else if (!strcmp(str, "trace")) |
584 | type = PERF_TYPE_TRACEPOINT; | ||
585 | else if (!strcmp(str, "raw")) | ||
586 | type = PERF_TYPE_RAW; | ||
587 | else { | ||
588 | fprintf(stderr, "Invalid event type in field string.\n"); | ||
589 | return -EINVAL; | ||
590 | } | ||
591 | |||
592 | if (output[type].user_set) | ||
593 | pr_warning("Overriding previous field request for %s events.\n", | ||
594 | event_type(type)); | ||
595 | |||
596 | output[type].fields = 0; | ||
597 | output[type].user_set = true; | ||
598 | output[type].wildcard_set = false; | ||
599 | |||
600 | } else { | ||
601 | tok = str; | ||
602 | if (strlen(str) == 0) { | ||
603 | fprintf(stderr, | ||
604 | "Cannot set fields to 'none' for all event types.\n"); | ||
605 | rc = -EINVAL; | ||
606 | goto out; | ||
607 | } | ||
608 | |||
609 | if (output_set_by_user()) | ||
610 | pr_warning("Overriding previous field request for all events.\n"); | ||
611 | |||
612 | for (j = 0; j < PERF_TYPE_MAX; ++j) { | ||
613 | output[j].fields = 0; | ||
614 | output[j].user_set = true; | ||
615 | output[j].wildcard_set = true; | ||
616 | } | ||
480 | } | 617 | } |
481 | 618 | ||
482 | output_fields[type] = 0; | 619 | tok = strtok(tok, ","); |
483 | while (1) { | 620 | while (tok) { |
484 | tok = strtok(NULL, ","); | ||
485 | if (!tok) | ||
486 | break; | ||
487 | for (i = 0; i < imax; ++i) { | 621 | for (i = 0; i < imax; ++i) { |
488 | if (strcmp(tok, all_output_options[i].str) == 0) { | 622 | if (strcmp(tok, all_output_options[i].str) == 0) |
489 | output_fields[type] |= all_output_options[i].field; | ||
490 | break; | 623 | break; |
491 | } | ||
492 | } | 624 | } |
493 | if (i == imax) { | 625 | if (i == imax) { |
494 | fprintf(stderr, "Invalid field requested."); | 626 | fprintf(stderr, "Invalid field requested.\n"); |
495 | rc = -EINVAL; | 627 | rc = -EINVAL; |
496 | break; | 628 | goto out; |
497 | } | 629 | } |
498 | } | ||
499 | 630 | ||
500 | if (output_fields[type] == 0) { | 631 | if (type == -1) { |
501 | pr_debug("No fields requested for %s type. " | 632 | /* add user option to all events types for |
502 | "Events will not be displayed\n", event_type(type)); | 633 | * which it is valid |
634 | */ | ||
635 | for (j = 0; j < PERF_TYPE_MAX; ++j) { | ||
636 | if (output[j].invalid_fields & all_output_options[i].field) { | ||
637 | pr_warning("\'%s\' not valid for %s events. Ignoring.\n", | ||
638 | all_output_options[i].str, event_type(j)); | ||
639 | } else | ||
640 | output[j].fields |= all_output_options[i].field; | ||
641 | } | ||
642 | } else { | ||
643 | if (output[type].invalid_fields & all_output_options[i].field) { | ||
644 | fprintf(stderr, "\'%s\' not valid for %s events.\n", | ||
645 | all_output_options[i].str, event_type(type)); | ||
646 | |||
647 | rc = -EINVAL; | ||
648 | goto out; | ||
649 | } | ||
650 | output[type].fields |= all_output_options[i].field; | ||
651 | } | ||
652 | |||
653 | tok = strtok(NULL, ","); | ||
503 | } | 654 | } |
504 | 655 | ||
505 | output_set_by_user = true; | 656 | if (type >= 0) { |
657 | if (output[type].fields == 0) { | ||
658 | pr_debug("No fields requested for %s type. " | ||
659 | "Events will not be displayed.\n", event_type(type)); | ||
660 | } | ||
661 | } | ||
506 | 662 | ||
663 | out: | ||
507 | free(str); | 664 | free(str); |
508 | return rc; | 665 | return rc; |
509 | } | 666 | } |
@@ -829,7 +986,7 @@ static const struct option options[] = { | |||
829 | OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory", | 986 | OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory", |
830 | "Look for files with symbols relative to this directory"), | 987 | "Look for files with symbols relative to this directory"), |
831 | OPT_CALLBACK('f', "fields", NULL, "str", | 988 | OPT_CALLBACK('f', "fields", NULL, "str", |
832 | "comma separated output fields prepend with 'type:'. Valid types: hw,sw,trace. Fields: comm,tid,pid,time,cpu,event,trace,sym", | 989 | "comma separated output fields prepend with 'type:'. Valid types: hw,sw,trace,raw. Fields: comm,tid,pid,time,cpu,event,trace,sym", |
833 | parse_output_fields), | 990 | parse_output_fields), |
834 | 991 | ||
835 | OPT_END() | 992 | OPT_END() |
@@ -1020,7 +1177,7 @@ int cmd_script(int argc, const char **argv, const char *prefix __used) | |||
1020 | struct stat perf_stat; | 1177 | struct stat perf_stat; |
1021 | int input; | 1178 | int input; |
1022 | 1179 | ||
1023 | if (output_set_by_user) { | 1180 | if (output_set_by_user()) { |
1024 | fprintf(stderr, | 1181 | fprintf(stderr, |
1025 | "custom fields not supported for generated scripts"); | 1182 | "custom fields not supported for generated scripts"); |
1026 | return -1; | 1183 | return -1; |
@@ -1060,6 +1217,11 @@ int cmd_script(int argc, const char **argv, const char *prefix __used) | |||
1060 | pr_debug("perf script started with script %s\n\n", script_name); | 1217 | pr_debug("perf script started with script %s\n\n", script_name); |
1061 | } | 1218 | } |
1062 | 1219 | ||
1220 | |||
1221 | err = perf_session__check_output_opt(session); | ||
1222 | if (err < 0) | ||
1223 | goto out; | ||
1224 | |||
1063 | err = __cmd_script(session); | 1225 | err = __cmd_script(session); |
1064 | 1226 | ||
1065 | perf_session__delete(session); | 1227 | perf_session__delete(session); |
diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 03f0e45f1479..a9f06715e44d 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c | |||
@@ -6,24 +6,28 @@ | |||
6 | * | 6 | * |
7 | * Sample output: | 7 | * Sample output: |
8 | 8 | ||
9 | $ perf stat ~/hackbench 10 | 9 | $ perf stat ./hackbench 10 |
10 | Time: 0.104 | ||
11 | 10 | ||
12 | Performance counter stats for '/home/mingo/hackbench': | 11 | Time: 0.118 |
13 | 12 | ||
14 | 1255.538611 task clock ticks # 10.143 CPU utilization factor | 13 | Performance counter stats for './hackbench 10': |
15 | 54011 context switches # 0.043 M/sec | ||
16 | 385 CPU migrations # 0.000 M/sec | ||
17 | 17755 pagefaults # 0.014 M/sec | ||
18 | 3808323185 CPU cycles # 3033.219 M/sec | ||
19 | 1575111190 instructions # 1254.530 M/sec | ||
20 | 17367895 cache references # 13.833 M/sec | ||
21 | 7674421 cache misses # 6.112 M/sec | ||
22 | 14 | ||
23 | Wall-clock time elapsed: 123.786620 msecs | 15 | 1708.761321 task-clock # 11.037 CPUs utilized |
16 | 41,190 context-switches # 0.024 M/sec | ||
17 | 6,735 CPU-migrations # 0.004 M/sec | ||
18 | 17,318 page-faults # 0.010 M/sec | ||
19 | 5,205,202,243 cycles # 3.046 GHz | ||
20 | 3,856,436,920 stalled-cycles-frontend # 74.09% frontend cycles idle | ||
21 | 1,600,790,871 stalled-cycles-backend # 30.75% backend cycles idle | ||
22 | 2,603,501,247 instructions # 0.50 insns per cycle | ||
23 | # 1.48 stalled cycles per insn | ||
24 | 484,357,498 branches # 283.455 M/sec | ||
25 | 6,388,934 branch-misses # 1.32% of all branches | ||
26 | |||
27 | 0.154822978 seconds time elapsed | ||
24 | 28 | ||
25 | * | 29 | * |
26 | * Copyright (C) 2008, Red Hat Inc, Ingo Molnar <mingo@redhat.com> | 30 | * Copyright (C) 2008-2011, Red Hat Inc, Ingo Molnar <mingo@redhat.com> |
27 | * | 31 | * |
28 | * Improvements and fixes by: | 32 | * Improvements and fixes by: |
29 | * | 33 | * |
@@ -46,6 +50,7 @@ | |||
46 | #include "util/evlist.h" | 50 | #include "util/evlist.h" |
47 | #include "util/evsel.h" | 51 | #include "util/evsel.h" |
48 | #include "util/debug.h" | 52 | #include "util/debug.h" |
53 | #include "util/color.h" | ||
49 | #include "util/header.h" | 54 | #include "util/header.h" |
50 | #include "util/cpumap.h" | 55 | #include "util/cpumap.h" |
51 | #include "util/thread.h" | 56 | #include "util/thread.h" |
@@ -65,14 +70,107 @@ static struct perf_event_attr default_attrs[] = { | |||
65 | { .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_PAGE_FAULTS }, | 70 | { .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_PAGE_FAULTS }, |
66 | 71 | ||
67 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES }, | 72 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES }, |
73 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_STALLED_CYCLES_FRONTEND }, | ||
74 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_STALLED_CYCLES_BACKEND }, | ||
68 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS }, | 75 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS }, |
69 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, | 76 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, |
70 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_MISSES }, | 77 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_MISSES }, |
71 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CACHE_REFERENCES }, | ||
72 | { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CACHE_MISSES }, | ||
73 | 78 | ||
74 | }; | 79 | }; |
75 | 80 | ||
81 | /* | ||
82 | * Detailed stats (-d), covering the L1 and last level data caches: | ||
83 | */ | ||
84 | static struct perf_event_attr detailed_attrs[] = { | ||
85 | |||
86 | { .type = PERF_TYPE_HW_CACHE, | ||
87 | .config = | ||
88 | PERF_COUNT_HW_CACHE_L1D << 0 | | ||
89 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
90 | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, | ||
91 | |||
92 | { .type = PERF_TYPE_HW_CACHE, | ||
93 | .config = | ||
94 | PERF_COUNT_HW_CACHE_L1D << 0 | | ||
95 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
96 | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, | ||
97 | |||
98 | { .type = PERF_TYPE_HW_CACHE, | ||
99 | .config = | ||
100 | PERF_COUNT_HW_CACHE_LL << 0 | | ||
101 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
102 | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, | ||
103 | |||
104 | { .type = PERF_TYPE_HW_CACHE, | ||
105 | .config = | ||
106 | PERF_COUNT_HW_CACHE_LL << 0 | | ||
107 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
108 | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, | ||
109 | }; | ||
110 | |||
111 | /* | ||
112 | * Very detailed stats (-d -d), covering the instruction cache and the TLB caches: | ||
113 | */ | ||
114 | static struct perf_event_attr very_detailed_attrs[] = { | ||
115 | |||
116 | { .type = PERF_TYPE_HW_CACHE, | ||
117 | .config = | ||
118 | PERF_COUNT_HW_CACHE_L1I << 0 | | ||
119 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
120 | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, | ||
121 | |||
122 | { .type = PERF_TYPE_HW_CACHE, | ||
123 | .config = | ||
124 | PERF_COUNT_HW_CACHE_L1I << 0 | | ||
125 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
126 | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, | ||
127 | |||
128 | { .type = PERF_TYPE_HW_CACHE, | ||
129 | .config = | ||
130 | PERF_COUNT_HW_CACHE_DTLB << 0 | | ||
131 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
132 | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, | ||
133 | |||
134 | { .type = PERF_TYPE_HW_CACHE, | ||
135 | .config = | ||
136 | PERF_COUNT_HW_CACHE_DTLB << 0 | | ||
137 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
138 | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, | ||
139 | |||
140 | { .type = PERF_TYPE_HW_CACHE, | ||
141 | .config = | ||
142 | PERF_COUNT_HW_CACHE_ITLB << 0 | | ||
143 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
144 | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, | ||
145 | |||
146 | { .type = PERF_TYPE_HW_CACHE, | ||
147 | .config = | ||
148 | PERF_COUNT_HW_CACHE_ITLB << 0 | | ||
149 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | | ||
150 | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, | ||
151 | |||
152 | }; | ||
153 | |||
154 | /* | ||
155 | * Very, very detailed stats (-d -d -d), adding prefetch events: | ||
156 | */ | ||
157 | static struct perf_event_attr very_very_detailed_attrs[] = { | ||
158 | |||
159 | { .type = PERF_TYPE_HW_CACHE, | ||
160 | .config = | ||
161 | PERF_COUNT_HW_CACHE_L1D << 0 | | ||
162 | (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) | | ||
163 | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, | ||
164 | |||
165 | { .type = PERF_TYPE_HW_CACHE, | ||
166 | .config = | ||
167 | PERF_COUNT_HW_CACHE_L1D << 0 | | ||
168 | (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) | | ||
169 | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, | ||
170 | }; | ||
171 | |||
172 | |||
173 | |||
76 | struct perf_evlist *evsel_list; | 174 | struct perf_evlist *evsel_list; |
77 | 175 | ||
78 | static bool system_wide = false; | 176 | static bool system_wide = false; |
@@ -86,6 +184,8 @@ static pid_t target_pid = -1; | |||
86 | static pid_t target_tid = -1; | 184 | static pid_t target_tid = -1; |
87 | static pid_t child_pid = -1; | 185 | static pid_t child_pid = -1; |
88 | static bool null_run = false; | 186 | static bool null_run = false; |
187 | static int detailed_run = 0; | ||
188 | static bool sync_run = false; | ||
89 | static bool big_num = true; | 189 | static bool big_num = true; |
90 | static int big_num_opt = -1; | 190 | static int big_num_opt = -1; |
91 | static const char *cpu_list; | 191 | static const char *cpu_list; |
@@ -156,7 +256,15 @@ static double stddev_stats(struct stats *stats) | |||
156 | 256 | ||
157 | struct stats runtime_nsecs_stats[MAX_NR_CPUS]; | 257 | struct stats runtime_nsecs_stats[MAX_NR_CPUS]; |
158 | struct stats runtime_cycles_stats[MAX_NR_CPUS]; | 258 | struct stats runtime_cycles_stats[MAX_NR_CPUS]; |
259 | struct stats runtime_stalled_cycles_front_stats[MAX_NR_CPUS]; | ||
260 | struct stats runtime_stalled_cycles_back_stats[MAX_NR_CPUS]; | ||
159 | struct stats runtime_branches_stats[MAX_NR_CPUS]; | 261 | struct stats runtime_branches_stats[MAX_NR_CPUS]; |
262 | struct stats runtime_cacherefs_stats[MAX_NR_CPUS]; | ||
263 | struct stats runtime_l1_dcache_stats[MAX_NR_CPUS]; | ||
264 | struct stats runtime_l1_icache_stats[MAX_NR_CPUS]; | ||
265 | struct stats runtime_ll_cache_stats[MAX_NR_CPUS]; | ||
266 | struct stats runtime_itlb_cache_stats[MAX_NR_CPUS]; | ||
267 | struct stats runtime_dtlb_cache_stats[MAX_NR_CPUS]; | ||
160 | struct stats walltime_nsecs_stats; | 268 | struct stats walltime_nsecs_stats; |
161 | 269 | ||
162 | static int create_perf_stat_counter(struct perf_evsel *evsel) | 270 | static int create_perf_stat_counter(struct perf_evsel *evsel) |
@@ -193,6 +301,37 @@ static inline int nsec_counter(struct perf_evsel *evsel) | |||
193 | } | 301 | } |
194 | 302 | ||
195 | /* | 303 | /* |
304 | * Update various tracking values we maintain to print | ||
305 | * more semantic information such as miss/hit ratios, | ||
306 | * instruction rates, etc: | ||
307 | */ | ||
308 | static void update_shadow_stats(struct perf_evsel *counter, u64 *count) | ||
309 | { | ||
310 | if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK)) | ||
311 | update_stats(&runtime_nsecs_stats[0], count[0]); | ||
312 | else if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES)) | ||
313 | update_stats(&runtime_cycles_stats[0], count[0]); | ||
314 | else if (perf_evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_FRONTEND)) | ||
315 | update_stats(&runtime_stalled_cycles_front_stats[0], count[0]); | ||
316 | else if (perf_evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_BACKEND)) | ||
317 | update_stats(&runtime_stalled_cycles_back_stats[0], count[0]); | ||
318 | else if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS)) | ||
319 | update_stats(&runtime_branches_stats[0], count[0]); | ||
320 | else if (perf_evsel__match(counter, HARDWARE, HW_CACHE_REFERENCES)) | ||
321 | update_stats(&runtime_cacherefs_stats[0], count[0]); | ||
322 | else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_L1D)) | ||
323 | update_stats(&runtime_l1_dcache_stats[0], count[0]); | ||
324 | else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_L1I)) | ||
325 | update_stats(&runtime_l1_icache_stats[0], count[0]); | ||
326 | else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_LL)) | ||
327 | update_stats(&runtime_ll_cache_stats[0], count[0]); | ||
328 | else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_DTLB)) | ||
329 | update_stats(&runtime_dtlb_cache_stats[0], count[0]); | ||
330 | else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_ITLB)) | ||
331 | update_stats(&runtime_itlb_cache_stats[0], count[0]); | ||
332 | } | ||
333 | |||
334 | /* | ||
196 | * Read out the results of a single counter: | 335 | * Read out the results of a single counter: |
197 | * aggregate counts across CPUs in system-wide mode | 336 | * aggregate counts across CPUs in system-wide mode |
198 | */ | 337 | */ |
@@ -217,12 +356,7 @@ static int read_counter_aggr(struct perf_evsel *counter) | |||
217 | /* | 356 | /* |
218 | * Save the full runtime - to allow normalization during printout: | 357 | * Save the full runtime - to allow normalization during printout: |
219 | */ | 358 | */ |
220 | if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK)) | 359 | update_shadow_stats(counter, count); |
221 | update_stats(&runtime_nsecs_stats[0], count[0]); | ||
222 | if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES)) | ||
223 | update_stats(&runtime_cycles_stats[0], count[0]); | ||
224 | if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS)) | ||
225 | update_stats(&runtime_branches_stats[0], count[0]); | ||
226 | 360 | ||
227 | return 0; | 361 | return 0; |
228 | } | 362 | } |
@@ -242,12 +376,7 @@ static int read_counter(struct perf_evsel *counter) | |||
242 | 376 | ||
243 | count = counter->counts->cpu[cpu].values; | 377 | count = counter->counts->cpu[cpu].values; |
244 | 378 | ||
245 | if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK)) | 379 | update_shadow_stats(counter, count); |
246 | update_stats(&runtime_nsecs_stats[cpu], count[0]); | ||
247 | if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES)) | ||
248 | update_stats(&runtime_cycles_stats[cpu], count[0]); | ||
249 | if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS)) | ||
250 | update_stats(&runtime_branches_stats[cpu], count[0]); | ||
251 | } | 380 | } |
252 | 381 | ||
253 | return 0; | 382 | return 0; |
@@ -315,13 +444,18 @@ static int run_perf_stat(int argc __used, const char **argv) | |||
315 | 444 | ||
316 | list_for_each_entry(counter, &evsel_list->entries, node) { | 445 | list_for_each_entry(counter, &evsel_list->entries, node) { |
317 | if (create_perf_stat_counter(counter) < 0) { | 446 | if (create_perf_stat_counter(counter) < 0) { |
318 | if (errno == -EPERM || errno == -EACCES) { | 447 | if (errno == EINVAL || errno == ENOSYS || errno == ENOENT) { |
448 | if (verbose) | ||
449 | ui__warning("%s event is not supported by the kernel.\n", | ||
450 | event_name(counter)); | ||
451 | continue; | ||
452 | } | ||
453 | |||
454 | if (errno == EPERM || errno == EACCES) { | ||
319 | error("You may not have permission to collect %sstats.\n" | 455 | error("You may not have permission to collect %sstats.\n" |
320 | "\t Consider tweaking" | 456 | "\t Consider tweaking" |
321 | " /proc/sys/kernel/perf_event_paranoid or running as root.", | 457 | " /proc/sys/kernel/perf_event_paranoid or running as root.", |
322 | system_wide ? "system-wide " : ""); | 458 | system_wide ? "system-wide " : ""); |
323 | } else if (errno == ENOENT) { | ||
324 | error("%s event is not supported. ", event_name(counter)); | ||
325 | } else { | 459 | } else { |
326 | error("open_counter returned with %d (%s). " | 460 | error("open_counter returned with %d (%s). " |
327 | "/bin/dmesg may provide additional information.\n", | 461 | "/bin/dmesg may provide additional information.\n", |
@@ -372,6 +506,16 @@ static int run_perf_stat(int argc __used, const char **argv) | |||
372 | return WEXITSTATUS(status); | 506 | return WEXITSTATUS(status); |
373 | } | 507 | } |
374 | 508 | ||
509 | static void print_noise_pct(double total, double avg) | ||
510 | { | ||
511 | double pct = 0.0; | ||
512 | |||
513 | if (avg) | ||
514 | pct = 100.0*total/avg; | ||
515 | |||
516 | fprintf(stderr, " ( +-%6.2f%% )", pct); | ||
517 | } | ||
518 | |||
375 | static void print_noise(struct perf_evsel *evsel, double avg) | 519 | static void print_noise(struct perf_evsel *evsel, double avg) |
376 | { | 520 | { |
377 | struct perf_stat *ps; | 521 | struct perf_stat *ps; |
@@ -380,15 +524,14 @@ static void print_noise(struct perf_evsel *evsel, double avg) | |||
380 | return; | 524 | return; |
381 | 525 | ||
382 | ps = evsel->priv; | 526 | ps = evsel->priv; |
383 | fprintf(stderr, " ( +- %7.3f%% )", | 527 | print_noise_pct(stddev_stats(&ps->res_stats[0]), avg); |
384 | 100 * stddev_stats(&ps->res_stats[0]) / avg); | ||
385 | } | 528 | } |
386 | 529 | ||
387 | static void nsec_printout(int cpu, struct perf_evsel *evsel, double avg) | 530 | static void nsec_printout(int cpu, struct perf_evsel *evsel, double avg) |
388 | { | 531 | { |
389 | double msecs = avg / 1e6; | 532 | double msecs = avg / 1e6; |
390 | char cpustr[16] = { '\0', }; | 533 | char cpustr[16] = { '\0', }; |
391 | const char *fmt = csv_output ? "%s%.6f%s%s" : "%s%18.6f%s%-24s"; | 534 | const char *fmt = csv_output ? "%s%.6f%s%s" : "%s%18.6f%s%-25s"; |
392 | 535 | ||
393 | if (no_aggr) | 536 | if (no_aggr) |
394 | sprintf(cpustr, "CPU%*d%s", | 537 | sprintf(cpustr, "CPU%*d%s", |
@@ -404,8 +547,191 @@ static void nsec_printout(int cpu, struct perf_evsel *evsel, double avg) | |||
404 | return; | 547 | return; |
405 | 548 | ||
406 | if (perf_evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK)) | 549 | if (perf_evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK)) |
407 | fprintf(stderr, " # %10.3f CPUs ", | 550 | fprintf(stderr, " # %8.3f CPUs utilized ", avg / avg_stats(&walltime_nsecs_stats)); |
408 | avg / avg_stats(&walltime_nsecs_stats)); | 551 | } |
552 | |||
553 | static void print_stalled_cycles_frontend(int cpu, struct perf_evsel *evsel __used, double avg) | ||
554 | { | ||
555 | double total, ratio = 0.0; | ||
556 | const char *color; | ||
557 | |||
558 | total = avg_stats(&runtime_cycles_stats[cpu]); | ||
559 | |||
560 | if (total) | ||
561 | ratio = avg / total * 100.0; | ||
562 | |||
563 | color = PERF_COLOR_NORMAL; | ||
564 | if (ratio > 50.0) | ||
565 | color = PERF_COLOR_RED; | ||
566 | else if (ratio > 30.0) | ||
567 | color = PERF_COLOR_MAGENTA; | ||
568 | else if (ratio > 10.0) | ||
569 | color = PERF_COLOR_YELLOW; | ||
570 | |||
571 | fprintf(stderr, " # "); | ||
572 | color_fprintf(stderr, color, "%6.2f%%", ratio); | ||
573 | fprintf(stderr, " frontend cycles idle "); | ||
574 | } | ||
575 | |||
576 | static void print_stalled_cycles_backend(int cpu, struct perf_evsel *evsel __used, double avg) | ||
577 | { | ||
578 | double total, ratio = 0.0; | ||
579 | const char *color; | ||
580 | |||
581 | total = avg_stats(&runtime_cycles_stats[cpu]); | ||
582 | |||
583 | if (total) | ||
584 | ratio = avg / total * 100.0; | ||
585 | |||
586 | color = PERF_COLOR_NORMAL; | ||
587 | if (ratio > 75.0) | ||
588 | color = PERF_COLOR_RED; | ||
589 | else if (ratio > 50.0) | ||
590 | color = PERF_COLOR_MAGENTA; | ||
591 | else if (ratio > 20.0) | ||
592 | color = PERF_COLOR_YELLOW; | ||
593 | |||
594 | fprintf(stderr, " # "); | ||
595 | color_fprintf(stderr, color, "%6.2f%%", ratio); | ||
596 | fprintf(stderr, " backend cycles idle "); | ||
597 | } | ||
598 | |||
599 | static void print_branch_misses(int cpu, struct perf_evsel *evsel __used, double avg) | ||
600 | { | ||
601 | double total, ratio = 0.0; | ||
602 | const char *color; | ||
603 | |||
604 | total = avg_stats(&runtime_branches_stats[cpu]); | ||
605 | |||
606 | if (total) | ||
607 | ratio = avg / total * 100.0; | ||
608 | |||
609 | color = PERF_COLOR_NORMAL; | ||
610 | if (ratio > 20.0) | ||
611 | color = PERF_COLOR_RED; | ||
612 | else if (ratio > 10.0) | ||
613 | color = PERF_COLOR_MAGENTA; | ||
614 | else if (ratio > 5.0) | ||
615 | color = PERF_COLOR_YELLOW; | ||
616 | |||
617 | fprintf(stderr, " # "); | ||
618 | color_fprintf(stderr, color, "%6.2f%%", ratio); | ||
619 | fprintf(stderr, " of all branches "); | ||
620 | } | ||
621 | |||
622 | static void print_l1_dcache_misses(int cpu, struct perf_evsel *evsel __used, double avg) | ||
623 | { | ||
624 | double total, ratio = 0.0; | ||
625 | const char *color; | ||
626 | |||
627 | total = avg_stats(&runtime_l1_dcache_stats[cpu]); | ||
628 | |||
629 | if (total) | ||
630 | ratio = avg / total * 100.0; | ||
631 | |||
632 | color = PERF_COLOR_NORMAL; | ||
633 | if (ratio > 20.0) | ||
634 | color = PERF_COLOR_RED; | ||
635 | else if (ratio > 10.0) | ||
636 | color = PERF_COLOR_MAGENTA; | ||
637 | else if (ratio > 5.0) | ||
638 | color = PERF_COLOR_YELLOW; | ||
639 | |||
640 | fprintf(stderr, " # "); | ||
641 | color_fprintf(stderr, color, "%6.2f%%", ratio); | ||
642 | fprintf(stderr, " of all L1-dcache hits "); | ||
643 | } | ||
644 | |||
645 | static void print_l1_icache_misses(int cpu, struct perf_evsel *evsel __used, double avg) | ||
646 | { | ||
647 | double total, ratio = 0.0; | ||
648 | const char *color; | ||
649 | |||
650 | total = avg_stats(&runtime_l1_icache_stats[cpu]); | ||
651 | |||
652 | if (total) | ||
653 | ratio = avg / total * 100.0; | ||
654 | |||
655 | color = PERF_COLOR_NORMAL; | ||
656 | if (ratio > 20.0) | ||
657 | color = PERF_COLOR_RED; | ||
658 | else if (ratio > 10.0) | ||
659 | color = PERF_COLOR_MAGENTA; | ||
660 | else if (ratio > 5.0) | ||
661 | color = PERF_COLOR_YELLOW; | ||
662 | |||
663 | fprintf(stderr, " # "); | ||
664 | color_fprintf(stderr, color, "%6.2f%%", ratio); | ||
665 | fprintf(stderr, " of all L1-icache hits "); | ||
666 | } | ||
667 | |||
668 | static void print_dtlb_cache_misses(int cpu, struct perf_evsel *evsel __used, double avg) | ||
669 | { | ||
670 | double total, ratio = 0.0; | ||
671 | const char *color; | ||
672 | |||
673 | total = avg_stats(&runtime_dtlb_cache_stats[cpu]); | ||
674 | |||
675 | if (total) | ||
676 | ratio = avg / total * 100.0; | ||
677 | |||
678 | color = PERF_COLOR_NORMAL; | ||
679 | if (ratio > 20.0) | ||
680 | color = PERF_COLOR_RED; | ||
681 | else if (ratio > 10.0) | ||
682 | color = PERF_COLOR_MAGENTA; | ||
683 | else if (ratio > 5.0) | ||
684 | color = PERF_COLOR_YELLOW; | ||
685 | |||
686 | fprintf(stderr, " # "); | ||
687 | color_fprintf(stderr, color, "%6.2f%%", ratio); | ||
688 | fprintf(stderr, " of all dTLB cache hits "); | ||
689 | } | ||
690 | |||
691 | static void print_itlb_cache_misses(int cpu, struct perf_evsel *evsel __used, double avg) | ||
692 | { | ||
693 | double total, ratio = 0.0; | ||
694 | const char *color; | ||
695 | |||
696 | total = avg_stats(&runtime_itlb_cache_stats[cpu]); | ||
697 | |||
698 | if (total) | ||
699 | ratio = avg / total * 100.0; | ||
700 | |||
701 | color = PERF_COLOR_NORMAL; | ||
702 | if (ratio > 20.0) | ||
703 | color = PERF_COLOR_RED; | ||
704 | else if (ratio > 10.0) | ||
705 | color = PERF_COLOR_MAGENTA; | ||
706 | else if (ratio > 5.0) | ||
707 | color = PERF_COLOR_YELLOW; | ||
708 | |||
709 | fprintf(stderr, " # "); | ||
710 | color_fprintf(stderr, color, "%6.2f%%", ratio); | ||
711 | fprintf(stderr, " of all iTLB cache hits "); | ||
712 | } | ||
713 | |||
714 | static void print_ll_cache_misses(int cpu, struct perf_evsel *evsel __used, double avg) | ||
715 | { | ||
716 | double total, ratio = 0.0; | ||
717 | const char *color; | ||
718 | |||
719 | total = avg_stats(&runtime_ll_cache_stats[cpu]); | ||
720 | |||
721 | if (total) | ||
722 | ratio = avg / total * 100.0; | ||
723 | |||
724 | color = PERF_COLOR_NORMAL; | ||
725 | if (ratio > 20.0) | ||
726 | color = PERF_COLOR_RED; | ||
727 | else if (ratio > 10.0) | ||
728 | color = PERF_COLOR_MAGENTA; | ||
729 | else if (ratio > 5.0) | ||
730 | color = PERF_COLOR_YELLOW; | ||
731 | |||
732 | fprintf(stderr, " # "); | ||
733 | color_fprintf(stderr, color, "%6.2f%%", ratio); | ||
734 | fprintf(stderr, " of all LL-cache hits "); | ||
409 | } | 735 | } |
410 | 736 | ||
411 | static void abs_printout(int cpu, struct perf_evsel *evsel, double avg) | 737 | static void abs_printout(int cpu, struct perf_evsel *evsel, double avg) |
@@ -417,9 +743,9 @@ static void abs_printout(int cpu, struct perf_evsel *evsel, double avg) | |||
417 | if (csv_output) | 743 | if (csv_output) |
418 | fmt = "%s%.0f%s%s"; | 744 | fmt = "%s%.0f%s%s"; |
419 | else if (big_num) | 745 | else if (big_num) |
420 | fmt = "%s%'18.0f%s%-24s"; | 746 | fmt = "%s%'18.0f%s%-25s"; |
421 | else | 747 | else |
422 | fmt = "%s%18.0f%s%-24s"; | 748 | fmt = "%s%18.0f%s%-25s"; |
423 | 749 | ||
424 | if (no_aggr) | 750 | if (no_aggr) |
425 | sprintf(cpustr, "CPU%*d%s", | 751 | sprintf(cpustr, "CPU%*d%s", |
@@ -442,23 +768,83 @@ static void abs_printout(int cpu, struct perf_evsel *evsel, double avg) | |||
442 | if (total) | 768 | if (total) |
443 | ratio = avg / total; | 769 | ratio = avg / total; |
444 | 770 | ||
445 | fprintf(stderr, " # %10.3f IPC ", ratio); | 771 | fprintf(stderr, " # %5.2f insns per cycle ", ratio); |
772 | |||
773 | total = avg_stats(&runtime_stalled_cycles_front_stats[cpu]); | ||
774 | total = max(total, avg_stats(&runtime_stalled_cycles_back_stats[cpu])); | ||
775 | |||
776 | if (total && avg) { | ||
777 | ratio = total / avg; | ||
778 | fprintf(stderr, "\n # %5.2f stalled cycles per insn", ratio); | ||
779 | } | ||
780 | |||
446 | } else if (perf_evsel__match(evsel, HARDWARE, HW_BRANCH_MISSES) && | 781 | } else if (perf_evsel__match(evsel, HARDWARE, HW_BRANCH_MISSES) && |
447 | runtime_branches_stats[cpu].n != 0) { | 782 | runtime_branches_stats[cpu].n != 0) { |
448 | total = avg_stats(&runtime_branches_stats[cpu]); | 783 | print_branch_misses(cpu, evsel, avg); |
784 | } else if ( | ||
785 | evsel->attr.type == PERF_TYPE_HW_CACHE && | ||
786 | evsel->attr.config == ( PERF_COUNT_HW_CACHE_L1D | | ||
787 | ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | | ||
788 | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) && | ||
789 | runtime_l1_dcache_stats[cpu].n != 0) { | ||
790 | print_l1_dcache_misses(cpu, evsel, avg); | ||
791 | } else if ( | ||
792 | evsel->attr.type == PERF_TYPE_HW_CACHE && | ||
793 | evsel->attr.config == ( PERF_COUNT_HW_CACHE_L1I | | ||
794 | ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | | ||
795 | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) && | ||
796 | runtime_l1_icache_stats[cpu].n != 0) { | ||
797 | print_l1_icache_misses(cpu, evsel, avg); | ||
798 | } else if ( | ||
799 | evsel->attr.type == PERF_TYPE_HW_CACHE && | ||
800 | evsel->attr.config == ( PERF_COUNT_HW_CACHE_DTLB | | ||
801 | ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | | ||
802 | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) && | ||
803 | runtime_dtlb_cache_stats[cpu].n != 0) { | ||
804 | print_dtlb_cache_misses(cpu, evsel, avg); | ||
805 | } else if ( | ||
806 | evsel->attr.type == PERF_TYPE_HW_CACHE && | ||
807 | evsel->attr.config == ( PERF_COUNT_HW_CACHE_ITLB | | ||
808 | ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | | ||
809 | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) && | ||
810 | runtime_itlb_cache_stats[cpu].n != 0) { | ||
811 | print_itlb_cache_misses(cpu, evsel, avg); | ||
812 | } else if ( | ||
813 | evsel->attr.type == PERF_TYPE_HW_CACHE && | ||
814 | evsel->attr.config == ( PERF_COUNT_HW_CACHE_LL | | ||
815 | ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | | ||
816 | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) && | ||
817 | runtime_ll_cache_stats[cpu].n != 0) { | ||
818 | print_ll_cache_misses(cpu, evsel, avg); | ||
819 | } else if (perf_evsel__match(evsel, HARDWARE, HW_CACHE_MISSES) && | ||
820 | runtime_cacherefs_stats[cpu].n != 0) { | ||
821 | total = avg_stats(&runtime_cacherefs_stats[cpu]); | ||
449 | 822 | ||
450 | if (total) | 823 | if (total) |
451 | ratio = avg * 100 / total; | 824 | ratio = avg * 100 / total; |
452 | 825 | ||
453 | fprintf(stderr, " # %10.3f %% ", ratio); | 826 | fprintf(stderr, " # %8.3f %% of all cache refs ", ratio); |
827 | |||
828 | } else if (perf_evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_FRONTEND)) { | ||
829 | print_stalled_cycles_frontend(cpu, evsel, avg); | ||
830 | } else if (perf_evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_BACKEND)) { | ||
831 | print_stalled_cycles_backend(cpu, evsel, avg); | ||
832 | } else if (perf_evsel__match(evsel, HARDWARE, HW_CPU_CYCLES)) { | ||
833 | total = avg_stats(&runtime_nsecs_stats[cpu]); | ||
454 | 834 | ||
835 | if (total) | ||
836 | ratio = 1.0 * avg / total; | ||
837 | |||
838 | fprintf(stderr, " # %8.3f GHz ", ratio); | ||
455 | } else if (runtime_nsecs_stats[cpu].n != 0) { | 839 | } else if (runtime_nsecs_stats[cpu].n != 0) { |
456 | total = avg_stats(&runtime_nsecs_stats[cpu]); | 840 | total = avg_stats(&runtime_nsecs_stats[cpu]); |
457 | 841 | ||
458 | if (total) | 842 | if (total) |
459 | ratio = 1000.0 * avg / total; | 843 | ratio = 1000.0 * avg / total; |
460 | 844 | ||
461 | fprintf(stderr, " # %10.3f M/sec", ratio); | 845 | fprintf(stderr, " # %8.3f M/sec ", ratio); |
846 | } else { | ||
847 | fprintf(stderr, " "); | ||
462 | } | 848 | } |
463 | } | 849 | } |
464 | 850 | ||
@@ -505,8 +891,7 @@ static void print_counter_aggr(struct perf_evsel *counter) | |||
505 | avg_enabled = avg_stats(&ps->res_stats[1]); | 891 | avg_enabled = avg_stats(&ps->res_stats[1]); |
506 | avg_running = avg_stats(&ps->res_stats[2]); | 892 | avg_running = avg_stats(&ps->res_stats[2]); |
507 | 893 | ||
508 | fprintf(stderr, " (scaled from %.2f%%)", | 894 | fprintf(stderr, " [%5.2f%%]", 100 * avg_running / avg_enabled); |
509 | 100 * avg_running / avg_enabled); | ||
510 | } | 895 | } |
511 | fprintf(stderr, "\n"); | 896 | fprintf(stderr, "\n"); |
512 | } | 897 | } |
@@ -548,10 +933,8 @@ static void print_counter(struct perf_evsel *counter) | |||
548 | if (!csv_output) { | 933 | if (!csv_output) { |
549 | print_noise(counter, 1.0); | 934 | print_noise(counter, 1.0); |
550 | 935 | ||
551 | if (run != ena) { | 936 | if (run != ena) |
552 | fprintf(stderr, " (scaled from %.2f%%)", | 937 | fprintf(stderr, " (%.2f%%)", 100.0 * run / ena); |
553 | 100.0 * run / ena); | ||
554 | } | ||
555 | } | 938 | } |
556 | fputc('\n', stderr); | 939 | fputc('\n', stderr); |
557 | } | 940 | } |
@@ -591,13 +974,14 @@ static void print_stat(int argc, const char **argv) | |||
591 | } | 974 | } |
592 | 975 | ||
593 | if (!csv_output) { | 976 | if (!csv_output) { |
594 | fprintf(stderr, "\n"); | 977 | if (!null_run) |
595 | fprintf(stderr, " %18.9f seconds time elapsed", | 978 | fprintf(stderr, "\n"); |
979 | fprintf(stderr, " %17.9f seconds time elapsed", | ||
596 | avg_stats(&walltime_nsecs_stats)/1e9); | 980 | avg_stats(&walltime_nsecs_stats)/1e9); |
597 | if (run_count > 1) { | 981 | if (run_count > 1) { |
598 | fprintf(stderr, " ( +- %7.3f%% )", | 982 | fprintf(stderr, " "); |
599 | 100*stddev_stats(&walltime_nsecs_stats) / | 983 | print_noise_pct(stddev_stats(&walltime_nsecs_stats), |
600 | avg_stats(&walltime_nsecs_stats)); | 984 | avg_stats(&walltime_nsecs_stats)); |
601 | } | 985 | } |
602 | fprintf(stderr, "\n\n"); | 986 | fprintf(stderr, "\n\n"); |
603 | } | 987 | } |
@@ -659,6 +1043,10 @@ static const struct option options[] = { | |||
659 | "repeat command and print average + stddev (max: 100)"), | 1043 | "repeat command and print average + stddev (max: 100)"), |
660 | OPT_BOOLEAN('n', "null", &null_run, | 1044 | OPT_BOOLEAN('n', "null", &null_run, |
661 | "null run - dont start any counters"), | 1045 | "null run - dont start any counters"), |
1046 | OPT_INCR('d', "detailed", &detailed_run, | ||
1047 | "detailed run - start a lot of events"), | ||
1048 | OPT_BOOLEAN('S', "sync", &sync_run, | ||
1049 | "call sync() before starting a run"), | ||
662 | OPT_CALLBACK_NOOPT('B', "big-num", NULL, NULL, | 1050 | OPT_CALLBACK_NOOPT('B', "big-num", NULL, NULL, |
663 | "print large numbers with thousands\' separators", | 1051 | "print large numbers with thousands\' separators", |
664 | stat__set_big_num), | 1052 | stat__set_big_num), |
@@ -674,6 +1062,70 @@ static const struct option options[] = { | |||
674 | OPT_END() | 1062 | OPT_END() |
675 | }; | 1063 | }; |
676 | 1064 | ||
1065 | /* | ||
1066 | * Add default attributes, if there were no attributes specified or | ||
1067 | * if -d/--detailed, -d -d or -d -d -d is used: | ||
1068 | */ | ||
1069 | static int add_default_attributes(void) | ||
1070 | { | ||
1071 | struct perf_evsel *pos; | ||
1072 | size_t attr_nr = 0; | ||
1073 | size_t c; | ||
1074 | |||
1075 | /* Set attrs if no event is selected and !null_run: */ | ||
1076 | if (null_run) | ||
1077 | return 0; | ||
1078 | |||
1079 | if (!evsel_list->nr_entries) { | ||
1080 | for (c = 0; c < ARRAY_SIZE(default_attrs); c++) { | ||
1081 | pos = perf_evsel__new(default_attrs + c, c + attr_nr); | ||
1082 | if (pos == NULL) | ||
1083 | return -1; | ||
1084 | perf_evlist__add(evsel_list, pos); | ||
1085 | } | ||
1086 | attr_nr += c; | ||
1087 | } | ||
1088 | |||
1089 | /* Detailed events get appended to the event list: */ | ||
1090 | |||
1091 | if (detailed_run < 1) | ||
1092 | return 0; | ||
1093 | |||
1094 | /* Append detailed run extra attributes: */ | ||
1095 | for (c = 0; c < ARRAY_SIZE(detailed_attrs); c++) { | ||
1096 | pos = perf_evsel__new(detailed_attrs + c, c + attr_nr); | ||
1097 | if (pos == NULL) | ||
1098 | return -1; | ||
1099 | perf_evlist__add(evsel_list, pos); | ||
1100 | } | ||
1101 | attr_nr += c; | ||
1102 | |||
1103 | if (detailed_run < 2) | ||
1104 | return 0; | ||
1105 | |||
1106 | /* Append very detailed run extra attributes: */ | ||
1107 | for (c = 0; c < ARRAY_SIZE(very_detailed_attrs); c++) { | ||
1108 | pos = perf_evsel__new(very_detailed_attrs + c, c + attr_nr); | ||
1109 | if (pos == NULL) | ||
1110 | return -1; | ||
1111 | perf_evlist__add(evsel_list, pos); | ||
1112 | } | ||
1113 | |||
1114 | if (detailed_run < 3) | ||
1115 | return 0; | ||
1116 | |||
1117 | /* Append very, very detailed run extra attributes: */ | ||
1118 | for (c = 0; c < ARRAY_SIZE(very_very_detailed_attrs); c++) { | ||
1119 | pos = perf_evsel__new(very_very_detailed_attrs + c, c + attr_nr); | ||
1120 | if (pos == NULL) | ||
1121 | return -1; | ||
1122 | perf_evlist__add(evsel_list, pos); | ||
1123 | } | ||
1124 | |||
1125 | |||
1126 | return 0; | ||
1127 | } | ||
1128 | |||
677 | int cmd_stat(int argc, const char **argv, const char *prefix __used) | 1129 | int cmd_stat(int argc, const char **argv, const char *prefix __used) |
678 | { | 1130 | { |
679 | struct perf_evsel *pos; | 1131 | struct perf_evsel *pos; |
@@ -719,17 +1171,8 @@ int cmd_stat(int argc, const char **argv, const char *prefix __used) | |||
719 | usage_with_options(stat_usage, options); | 1171 | usage_with_options(stat_usage, options); |
720 | } | 1172 | } |
721 | 1173 | ||
722 | /* Set attrs and nr_counters if no event is selected and !null_run */ | 1174 | if (add_default_attributes()) |
723 | if (!null_run && !evsel_list->nr_entries) { | 1175 | goto out; |
724 | size_t c; | ||
725 | |||
726 | for (c = 0; c < ARRAY_SIZE(default_attrs); ++c) { | ||
727 | pos = perf_evsel__new(&default_attrs[c], c); | ||
728 | if (pos == NULL) | ||
729 | goto out; | ||
730 | perf_evlist__add(evsel_list, pos); | ||
731 | } | ||
732 | } | ||
733 | 1176 | ||
734 | if (target_pid != -1) | 1177 | if (target_pid != -1) |
735 | target_tid = target_pid; | 1178 | target_tid = target_pid; |
@@ -773,6 +1216,10 @@ int cmd_stat(int argc, const char **argv, const char *prefix __used) | |||
773 | for (run_idx = 0; run_idx < run_count; run_idx++) { | 1216 | for (run_idx = 0; run_idx < run_count; run_idx++) { |
774 | if (run_count != 1 && verbose) | 1217 | if (run_count != 1 && verbose) |
775 | fprintf(stderr, "[ perf stat: executing run #%d ... ]\n", run_idx + 1); | 1218 | fprintf(stderr, "[ perf stat: executing run #%d ... ]\n", run_idx + 1); |
1219 | |||
1220 | if (sync_run) | ||
1221 | sync(); | ||
1222 | |||
776 | status = run_perf_stat(argc, argv); | 1223 | status = run_perf_stat(argc, argv); |
777 | } | 1224 | } |
778 | 1225 | ||
diff --git a/tools/perf/feature-tests.mak b/tools/perf/config/feature-tests.mak index b041ca67a2cb..6170fd2531b5 100644 --- a/tools/perf/feature-tests.mak +++ b/tools/perf/config/feature-tests.mak | |||
@@ -79,9 +79,15 @@ endef | |||
79 | endif | 79 | endif |
80 | 80 | ||
81 | ifndef NO_LIBPYTHON | 81 | ifndef NO_LIBPYTHON |
82 | define SOURCE_PYTHON_VERSION | ||
83 | #include <Python.h> | ||
84 | #if PY_VERSION_HEX >= 0x03000000 | ||
85 | #error | ||
86 | #endif | ||
87 | int main(void){} | ||
88 | endef | ||
82 | define SOURCE_PYTHON_EMBED | 89 | define SOURCE_PYTHON_EMBED |
83 | #include <Python.h> | 90 | #include <Python.h> |
84 | |||
85 | int main(void) | 91 | int main(void) |
86 | { | 92 | { |
87 | Py_Initialize(); | 93 | Py_Initialize(); |
@@ -120,11 +126,3 @@ int main(void) | |||
120 | return 0; | 126 | return 0; |
121 | } | 127 | } |
122 | endef | 128 | endef |
123 | |||
124 | # try-cc | ||
125 | # Usage: option = $(call try-cc, source-to-build, cc-options) | ||
126 | try-cc = $(shell sh -c \ | ||
127 | 'TMP="$(OUTPUT)$(TMPOUT).$$$$"; \ | ||
128 | echo "$(1)" | \ | ||
129 | $(CC) -x c - $(2) -o "$$TMP" > /dev/null 2>&1 && echo y; \ | ||
130 | rm -f "$$TMP"') | ||
diff --git a/tools/perf/config/utilities.mak b/tools/perf/config/utilities.mak new file mode 100644 index 000000000000..8046182a19eb --- /dev/null +++ b/tools/perf/config/utilities.mak | |||
@@ -0,0 +1,188 @@ | |||
1 | # This allows us to work with the newline character: | ||
2 | define newline | ||
3 | |||
4 | |||
5 | endef | ||
6 | newline := $(newline) | ||
7 | |||
8 | # nl-escape | ||
9 | # | ||
10 | # Usage: escape = $(call nl-escape[,escape]) | ||
11 | # | ||
12 | # This is used as the common way to specify | ||
13 | # what should replace a newline when escaping | ||
14 | # newlines; the default is a bizarre string. | ||
15 | # | ||
16 | nl-escape = $(or $(1),m822df3020w6a44id34bt574ctac44eb9f4n) | ||
17 | |||
18 | # escape-nl | ||
19 | # | ||
20 | # Usage: escaped-text = $(call escape-nl,text[,escape]) | ||
21 | # | ||
22 | # GNU make's $(shell ...) function converts to a | ||
23 | # single space each newline character in the output | ||
24 | # produced during the expansion; this may not be | ||
25 | # desirable. | ||
26 | # | ||
27 | # The only solution is to change each newline into | ||
28 | # something that won't be converted, so that the | ||
29 | # information can be recovered later with | ||
30 | # $(call unescape-nl...) | ||
31 | # | ||
32 | escape-nl = $(subst $(newline),$(call nl-escape,$(2)),$(1)) | ||
33 | |||
34 | # unescape-nl | ||
35 | # | ||
36 | # Usage: text = $(call unescape-nl,escaped-text[,escape]) | ||
37 | # | ||
38 | # See escape-nl. | ||
39 | # | ||
40 | unescape-nl = $(subst $(call nl-escape,$(2)),$(newline),$(1)) | ||
41 | |||
42 | # shell-escape-nl | ||
43 | # | ||
44 | # Usage: $(shell some-command | $(call shell-escape-nl[,escape])) | ||
45 | # | ||
46 | # Use this to escape newlines from within a shell call; | ||
47 | # the default escape is a bizarre string. | ||
48 | # | ||
49 | # NOTE: The escape is used directly as a string constant | ||
50 | # in an `awk' program that is delimited by shell | ||
51 | # single-quotes, so be wary of the characters | ||
52 | # that are chosen. | ||
53 | # | ||
54 | define shell-escape-nl | ||
55 | awk 'NR==1 {t=$$0} NR>1 {t=t "$(nl-escape)" $$0} END {printf t}' | ||
56 | endef | ||
57 | |||
58 | # shell-unescape-nl | ||
59 | # | ||
60 | # Usage: $(shell some-command | $(call shell-unescape-nl[,escape])) | ||
61 | # | ||
62 | # Use this to unescape newlines from within a shell call; | ||
63 | # the default escape is a bizarre string. | ||
64 | # | ||
65 | # NOTE: The escape is used directly as an extended regular | ||
66 | # expression constant in an `awk' program that is | ||
67 | # delimited by shell single-quotes, so be wary | ||
68 | # of the characters that are chosen. | ||
69 | # | ||
70 | # (The bash shell has a bug where `{gsub(...),...}' is | ||
71 | # misinterpreted as a brace expansion; this can be | ||
72 | # overcome by putting a space between `{' and `gsub'). | ||
73 | # | ||
74 | define shell-unescape-nl | ||
75 | awk 'NR==1 {t=$$0} NR>1 {t=t "\n" $$0} END { gsub(/$(nl-escape)/,"\n",t); printf t }' | ||
76 | endef | ||
77 | |||
78 | # escape-for-shell-sq | ||
79 | # | ||
80 | # Usage: embeddable-text = $(call escape-for-shell-sq,text) | ||
81 | # | ||
82 | # This function produces text that is suitable for | ||
83 | # embedding in a shell string that is delimited by | ||
84 | # single-quotes. | ||
85 | # | ||
86 | escape-for-shell-sq = $(subst ','\'',$(1)) | ||
87 | |||
88 | # shell-sq | ||
89 | # | ||
90 | # Usage: single-quoted-and-escaped-text = $(call shell-sq,text) | ||
91 | # | ||
92 | shell-sq = '$(escape-for-shell-sq)' | ||
93 | |||
94 | # shell-wordify | ||
95 | # | ||
96 | # Usage: wordified-text = $(call shell-wordify,text) | ||
97 | # | ||
98 | # For instance: | ||
99 | # | ||
100 | # |define text | ||
101 | # |hello | ||
102 | # |world | ||
103 | # |endef | ||
104 | # | | ||
105 | # |target: | ||
106 | # | echo $(call shell-wordify,$(text)) | ||
107 | # | ||
108 | # At least GNU make gets confused by expanding a newline | ||
109 | # within the context of a command line of a makefile rule | ||
110 | # (this is in constrast to a `$(shell ...)' function call, | ||
111 | # which can handle it just fine). | ||
112 | # | ||
113 | # This function avoids the problem by producing a string | ||
114 | # that works as a shell word, regardless of whether or | ||
115 | # not it contains a newline. | ||
116 | # | ||
117 | # If the text to be wordified contains a newline, then | ||
118 | # an intrictate shell command substitution is constructed | ||
119 | # to render the text as a single line; when the shell | ||
120 | # processes the resulting escaped text, it transforms | ||
121 | # it into the original unescaped text. | ||
122 | # | ||
123 | # If the text does not contain a newline, then this function | ||
124 | # produces the same results as the `$(shell-sq)' function. | ||
125 | # | ||
126 | shell-wordify = $(if $(findstring $(newline),$(1)),$(_sw-esc-nl),$(shell-sq)) | ||
127 | define _sw-esc-nl | ||
128 | "$$(echo $(call escape-nl,$(shell-sq),$(2)) | $(call shell-unescape-nl,$(2)))" | ||
129 | endef | ||
130 | |||
131 | # is-absolute | ||
132 | # | ||
133 | # Usage: bool-value = $(call is-absolute,path) | ||
134 | # | ||
135 | is-absolute = $(shell echo $(shell-sq) | grep ^/ -q && echo y) | ||
136 | |||
137 | # lookup | ||
138 | # | ||
139 | # Usage: absolute-executable-path-or-empty = $(call lookup,path) | ||
140 | # | ||
141 | # (It's necessary to use `sh -c' because GNU make messes up by | ||
142 | # trying too hard and getting things wrong). | ||
143 | # | ||
144 | lookup = $(call unescape-nl,$(shell sh -c $(_l-sh))) | ||
145 | _l-sh = $(call shell-sq,command -v $(shell-sq) | $(call shell-escape-nl,)) | ||
146 | |||
147 | # is-executable | ||
148 | # | ||
149 | # Usage: bool-value = $(call is-executable,path) | ||
150 | # | ||
151 | # (It's necessary to use `sh -c' because GNU make messes up by | ||
152 | # trying too hard and getting things wrong). | ||
153 | # | ||
154 | is-executable = $(call _is-executable-helper,$(shell-sq)) | ||
155 | _is-executable-helper = $(shell sh -c $(_is-executable-sh)) | ||
156 | _is-executable-sh = $(call shell-sq,test -f $(1) -a -x $(1) && echo y) | ||
157 | |||
158 | # get-executable | ||
159 | # | ||
160 | # Usage: absolute-executable-path-or-empty = $(call get-executable,path) | ||
161 | # | ||
162 | # The goal is to get an absolute path for an executable; | ||
163 | # the `command -v' is defined by POSIX, but it's not | ||
164 | # necessarily very portable, so it's only used if | ||
165 | # relative path resolution is requested, as determined | ||
166 | # by the presence of a leading `/'. | ||
167 | # | ||
168 | get-executable = $(if $(1),$(if $(is-absolute),$(_ge-abspath),$(lookup))) | ||
169 | _ge-abspath = $(if $(is-executable),$(1)) | ||
170 | |||
171 | # get-supplied-or-default-executable | ||
172 | # | ||
173 | # Usage: absolute-executable-path-or-empty = $(call get-executable-or-default,variable,default) | ||
174 | # | ||
175 | define get-executable-or-default | ||
176 | $(if $($(1)),$(call _ge_attempt,$($(1)),$(1)),$(call _ge_attempt,$(2))) | ||
177 | endef | ||
178 | _ge_attempt = $(or $(get-executable),$(_gea_warn),$(call _gea_err,$(2))) | ||
179 | _gea_warn = $(warning The path '$(1)' is not executable.) | ||
180 | _gea_err = $(if $(1),$(error Please set '$(1)' appropriately)) | ||
181 | |||
182 | # try-cc | ||
183 | # Usage: option = $(call try-cc, source-to-build, cc-options) | ||
184 | try-cc = $(shell sh -c \ | ||
185 | 'TMP="$(OUTPUT)$(TMPOUT).$$$$"; \ | ||
186 | echo "$(1)" | \ | ||
187 | $(CC) -x c - $(2) -o "$$TMP" > /dev/null 2>&1 && echo y; \ | ||
188 | rm -f "$$TMP"') | ||
diff --git a/tools/perf/util/include/asm/alternative-asm.h b/tools/perf/util/include/asm/alternative-asm.h new file mode 100644 index 000000000000..6789d788d494 --- /dev/null +++ b/tools/perf/util/include/asm/alternative-asm.h | |||
@@ -0,0 +1,8 @@ | |||
1 | #ifndef _PERF_ASM_ALTERNATIVE_ASM_H | ||
2 | #define _PERF_ASM_ALTERNATIVE_ASM_H | ||
3 | |||
4 | /* Just disable it so we can build arch/x86/lib/memcpy_64.S for perf bench: */ | ||
5 | |||
6 | #define altinstruction_entry # | ||
7 | |||
8 | #endif | ||
diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 952b4ae3d954..41982c373faf 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c | |||
@@ -31,34 +31,36 @@ char debugfs_path[MAXPATHLEN]; | |||
31 | #define CSW(x) .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_##x | 31 | #define CSW(x) .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_##x |
32 | 32 | ||
33 | static struct event_symbol event_symbols[] = { | 33 | static struct event_symbol event_symbols[] = { |
34 | { CHW(CPU_CYCLES), "cpu-cycles", "cycles" }, | 34 | { CHW(CPU_CYCLES), "cpu-cycles", "cycles" }, |
35 | { CHW(INSTRUCTIONS), "instructions", "" }, | 35 | { CHW(STALLED_CYCLES_FRONTEND), "stalled-cycles-frontend", "idle-cycles-frontend" }, |
36 | { CHW(CACHE_REFERENCES), "cache-references", "" }, | 36 | { CHW(STALLED_CYCLES_BACKEND), "stalled-cycles-backend", "idle-cycles-backend" }, |
37 | { CHW(CACHE_MISSES), "cache-misses", "" }, | 37 | { CHW(INSTRUCTIONS), "instructions", "" }, |
38 | { CHW(BRANCH_INSTRUCTIONS), "branch-instructions", "branches" }, | 38 | { CHW(CACHE_REFERENCES), "cache-references", "" }, |
39 | { CHW(BRANCH_MISSES), "branch-misses", "" }, | 39 | { CHW(CACHE_MISSES), "cache-misses", "" }, |
40 | { CHW(BUS_CYCLES), "bus-cycles", "" }, | 40 | { CHW(BRANCH_INSTRUCTIONS), "branch-instructions", "branches" }, |
41 | 41 | { CHW(BRANCH_MISSES), "branch-misses", "" }, | |
42 | { CSW(CPU_CLOCK), "cpu-clock", "" }, | 42 | { CHW(BUS_CYCLES), "bus-cycles", "" }, |
43 | { CSW(TASK_CLOCK), "task-clock", "" }, | 43 | |
44 | { CSW(PAGE_FAULTS), "page-faults", "faults" }, | 44 | { CSW(CPU_CLOCK), "cpu-clock", "" }, |
45 | { CSW(PAGE_FAULTS_MIN), "minor-faults", "" }, | 45 | { CSW(TASK_CLOCK), "task-clock", "" }, |
46 | { CSW(PAGE_FAULTS_MAJ), "major-faults", "" }, | 46 | { CSW(PAGE_FAULTS), "page-faults", "faults" }, |
47 | { CSW(CONTEXT_SWITCHES), "context-switches", "cs" }, | 47 | { CSW(PAGE_FAULTS_MIN), "minor-faults", "" }, |
48 | { CSW(CPU_MIGRATIONS), "cpu-migrations", "migrations" }, | 48 | { CSW(PAGE_FAULTS_MAJ), "major-faults", "" }, |
49 | { CSW(ALIGNMENT_FAULTS), "alignment-faults", "" }, | 49 | { CSW(CONTEXT_SWITCHES), "context-switches", "cs" }, |
50 | { CSW(EMULATION_FAULTS), "emulation-faults", "" }, | 50 | { CSW(CPU_MIGRATIONS), "cpu-migrations", "migrations" }, |
51 | { CSW(ALIGNMENT_FAULTS), "alignment-faults", "" }, | ||
52 | { CSW(EMULATION_FAULTS), "emulation-faults", "" }, | ||
51 | }; | 53 | }; |
52 | 54 | ||
53 | #define __PERF_EVENT_FIELD(config, name) \ | 55 | #define __PERF_EVENT_FIELD(config, name) \ |
54 | ((config & PERF_EVENT_##name##_MASK) >> PERF_EVENT_##name##_SHIFT) | 56 | ((config & PERF_EVENT_##name##_MASK) >> PERF_EVENT_##name##_SHIFT) |
55 | 57 | ||
56 | #define PERF_EVENT_RAW(config) __PERF_EVENT_FIELD(config, RAW) | 58 | #define PERF_EVENT_RAW(config) __PERF_EVENT_FIELD(config, RAW) |
57 | #define PERF_EVENT_CONFIG(config) __PERF_EVENT_FIELD(config, CONFIG) | 59 | #define PERF_EVENT_CONFIG(config) __PERF_EVENT_FIELD(config, CONFIG) |
58 | #define PERF_EVENT_TYPE(config) __PERF_EVENT_FIELD(config, TYPE) | 60 | #define PERF_EVENT_TYPE(config) __PERF_EVENT_FIELD(config, TYPE) |
59 | #define PERF_EVENT_ID(config) __PERF_EVENT_FIELD(config, EVENT) | 61 | #define PERF_EVENT_ID(config) __PERF_EVENT_FIELD(config, EVENT) |
60 | 62 | ||
61 | static const char *hw_event_names[] = { | 63 | static const char *hw_event_names[PERF_COUNT_HW_MAX] = { |
62 | "cycles", | 64 | "cycles", |
63 | "instructions", | 65 | "instructions", |
64 | "cache-references", | 66 | "cache-references", |
@@ -66,11 +68,13 @@ static const char *hw_event_names[] = { | |||
66 | "branches", | 68 | "branches", |
67 | "branch-misses", | 69 | "branch-misses", |
68 | "bus-cycles", | 70 | "bus-cycles", |
71 | "stalled-cycles-frontend", | ||
72 | "stalled-cycles-backend", | ||
69 | }; | 73 | }; |
70 | 74 | ||
71 | static const char *sw_event_names[] = { | 75 | static const char *sw_event_names[PERF_COUNT_SW_MAX] = { |
72 | "cpu-clock-msecs", | 76 | "cpu-clock", |
73 | "task-clock-msecs", | 77 | "task-clock", |
74 | "page-faults", | 78 | "page-faults", |
75 | "context-switches", | 79 | "context-switches", |
76 | "CPU-migrations", | 80 | "CPU-migrations", |
@@ -307,7 +311,7 @@ const char *__event_name(int type, u64 config) | |||
307 | 311 | ||
308 | switch (type) { | 312 | switch (type) { |
309 | case PERF_TYPE_HARDWARE: | 313 | case PERF_TYPE_HARDWARE: |
310 | if (config < PERF_COUNT_HW_MAX) | 314 | if (config < PERF_COUNT_HW_MAX && hw_event_names[config]) |
311 | return hw_event_names[config]; | 315 | return hw_event_names[config]; |
312 | return "unknown-hardware"; | 316 | return "unknown-hardware"; |
313 | 317 | ||
@@ -333,7 +337,7 @@ const char *__event_name(int type, u64 config) | |||
333 | } | 337 | } |
334 | 338 | ||
335 | case PERF_TYPE_SOFTWARE: | 339 | case PERF_TYPE_SOFTWARE: |
336 | if (config < PERF_COUNT_SW_MAX) | 340 | if (config < PERF_COUNT_SW_MAX && sw_event_names[config]) |
337 | return sw_event_names[config]; | 341 | return sw_event_names[config]; |
338 | return "unknown-software"; | 342 | return "unknown-software"; |
339 | 343 | ||
@@ -648,13 +652,15 @@ static int check_events(const char *str, unsigned int i) | |||
648 | int n; | 652 | int n; |
649 | 653 | ||
650 | n = strlen(event_symbols[i].symbol); | 654 | n = strlen(event_symbols[i].symbol); |
651 | if (!strncmp(str, event_symbols[i].symbol, n)) | 655 | if (!strncasecmp(str, event_symbols[i].symbol, n)) |
652 | return n; | 656 | return n; |
653 | 657 | ||
654 | n = strlen(event_symbols[i].alias); | 658 | n = strlen(event_symbols[i].alias); |
655 | if (n) | 659 | if (n) { |
656 | if (!strncmp(str, event_symbols[i].alias, n)) | 660 | if (!strncasecmp(str, event_symbols[i].alias, n)) |
657 | return n; | 661 | return n; |
662 | } | ||
663 | |||
658 | return 0; | 664 | return 0; |
659 | } | 665 | } |
660 | 666 | ||
@@ -718,15 +724,22 @@ parse_numeric_event(const char **strp, struct perf_event_attr *attr) | |||
718 | return EVT_FAILED; | 724 | return EVT_FAILED; |
719 | } | 725 | } |
720 | 726 | ||
721 | static enum event_result | 727 | static int |
722 | parse_event_modifier(const char **strp, struct perf_event_attr *attr) | 728 | parse_event_modifier(const char **strp, struct perf_event_attr *attr) |
723 | { | 729 | { |
724 | const char *str = *strp; | 730 | const char *str = *strp; |
725 | int exclude = 0; | 731 | int exclude = 0; |
726 | int eu = 0, ek = 0, eh = 0, precise = 0; | 732 | int eu = 0, ek = 0, eh = 0, precise = 0; |
727 | 733 | ||
728 | if (*str++ != ':') | 734 | if (!*str) |
735 | return 0; | ||
736 | |||
737 | if (*str == ',') | ||
729 | return 0; | 738 | return 0; |
739 | |||
740 | if (*str++ != ':') | ||
741 | return -1; | ||
742 | |||
730 | while (*str) { | 743 | while (*str) { |
731 | if (*str == 'u') { | 744 | if (*str == 'u') { |
732 | if (!exclude) | 745 | if (!exclude) |
@@ -747,14 +760,16 @@ parse_event_modifier(const char **strp, struct perf_event_attr *attr) | |||
747 | 760 | ||
748 | ++str; | 761 | ++str; |
749 | } | 762 | } |
750 | if (str >= *strp + 2) { | 763 | if (str < *strp + 2) |
751 | *strp = str; | 764 | return -1; |
752 | attr->exclude_user = eu; | 765 | |
753 | attr->exclude_kernel = ek; | 766 | *strp = str; |
754 | attr->exclude_hv = eh; | 767 | |
755 | attr->precise_ip = precise; | 768 | attr->exclude_user = eu; |
756 | return 1; | 769 | attr->exclude_kernel = ek; |
757 | } | 770 | attr->exclude_hv = eh; |
771 | attr->precise_ip = precise; | ||
772 | |||
758 | return 0; | 773 | return 0; |
759 | } | 774 | } |
760 | 775 | ||
@@ -797,7 +812,12 @@ parse_event_symbols(const struct option *opt, const char **str, | |||
797 | return EVT_FAILED; | 812 | return EVT_FAILED; |
798 | 813 | ||
799 | modifier: | 814 | modifier: |
800 | parse_event_modifier(str, attr); | 815 | if (parse_event_modifier(str, attr) < 0) { |
816 | fprintf(stderr, "invalid event modifier: '%s'\n", *str); | ||
817 | fprintf(stderr, "Run 'perf list' for a list of valid events and modifiers\n"); | ||
818 | |||
819 | return EVT_FAILED; | ||
820 | } | ||
801 | 821 | ||
802 | return ret; | 822 | return ret; |
803 | } | 823 | } |
@@ -912,7 +932,7 @@ void print_tracepoint_events(const char *subsys_glob, const char *event_glob) | |||
912 | 932 | ||
913 | snprintf(evt_path, MAXPATHLEN, "%s:%s", | 933 | snprintf(evt_path, MAXPATHLEN, "%s:%s", |
914 | sys_dirent.d_name, evt_dirent.d_name); | 934 | sys_dirent.d_name, evt_dirent.d_name); |
915 | printf(" %-42s [%s]\n", evt_path, | 935 | printf(" %-50s [%s]\n", evt_path, |
916 | event_type_descriptors[PERF_TYPE_TRACEPOINT]); | 936 | event_type_descriptors[PERF_TYPE_TRACEPOINT]); |
917 | } | 937 | } |
918 | closedir(evt_dir); | 938 | closedir(evt_dir); |
@@ -977,7 +997,7 @@ void print_events_type(u8 type) | |||
977 | else | 997 | else |
978 | snprintf(name, sizeof(name), "%s", syms->symbol); | 998 | snprintf(name, sizeof(name), "%s", syms->symbol); |
979 | 999 | ||
980 | printf(" %-42s [%s]\n", name, | 1000 | printf(" %-50s [%s]\n", name, |
981 | event_type_descriptors[type]); | 1001 | event_type_descriptors[type]); |
982 | } | 1002 | } |
983 | } | 1003 | } |
@@ -995,11 +1015,10 @@ int print_hwcache_events(const char *event_glob) | |||
995 | for (i = 0; i < PERF_COUNT_HW_CACHE_RESULT_MAX; i++) { | 1015 | for (i = 0; i < PERF_COUNT_HW_CACHE_RESULT_MAX; i++) { |
996 | char *name = event_cache_name(type, op, i); | 1016 | char *name = event_cache_name(type, op, i); |
997 | 1017 | ||
998 | if (event_glob != NULL && | 1018 | if (event_glob != NULL && !strglobmatch(name, event_glob)) |
999 | !strglobmatch(name, event_glob)) | ||
1000 | continue; | 1019 | continue; |
1001 | 1020 | ||
1002 | printf(" %-42s [%s]\n", name, | 1021 | printf(" %-50s [%s]\n", name, |
1003 | event_type_descriptors[PERF_TYPE_HW_CACHE]); | 1022 | event_type_descriptors[PERF_TYPE_HW_CACHE]); |
1004 | ++printed; | 1023 | ++printed; |
1005 | } | 1024 | } |
@@ -1009,14 +1028,16 @@ int print_hwcache_events(const char *event_glob) | |||
1009 | return printed; | 1028 | return printed; |
1010 | } | 1029 | } |
1011 | 1030 | ||
1031 | #define MAX_NAME_LEN 100 | ||
1032 | |||
1012 | /* | 1033 | /* |
1013 | * Print the help text for the event symbols: | 1034 | * Print the help text for the event symbols: |
1014 | */ | 1035 | */ |
1015 | void print_events(const char *event_glob) | 1036 | void print_events(const char *event_glob) |
1016 | { | 1037 | { |
1017 | struct event_symbol *syms = event_symbols; | ||
1018 | unsigned int i, type, prev_type = -1, printed = 0, ntypes_printed = 0; | 1038 | unsigned int i, type, prev_type = -1, printed = 0, ntypes_printed = 0; |
1019 | char name[40]; | 1039 | struct event_symbol *syms = event_symbols; |
1040 | char name[MAX_NAME_LEN]; | ||
1020 | 1041 | ||
1021 | printf("\n"); | 1042 | printf("\n"); |
1022 | printf("List of pre-defined events (to be used in -e):\n"); | 1043 | printf("List of pre-defined events (to be used in -e):\n"); |
@@ -1036,10 +1057,10 @@ void print_events(const char *event_glob) | |||
1036 | continue; | 1057 | continue; |
1037 | 1058 | ||
1038 | if (strlen(syms->alias)) | 1059 | if (strlen(syms->alias)) |
1039 | sprintf(name, "%s OR %s", syms->symbol, syms->alias); | 1060 | snprintf(name, MAX_NAME_LEN, "%s OR %s", syms->symbol, syms->alias); |
1040 | else | 1061 | else |
1041 | strcpy(name, syms->symbol); | 1062 | strncpy(name, syms->symbol, MAX_NAME_LEN); |
1042 | printf(" %-42s [%s]\n", name, | 1063 | printf(" %-50s [%s]\n", name, |
1043 | event_type_descriptors[type]); | 1064 | event_type_descriptors[type]); |
1044 | 1065 | ||
1045 | prev_type = type; | 1066 | prev_type = type; |
@@ -1056,12 +1077,12 @@ void print_events(const char *event_glob) | |||
1056 | return; | 1077 | return; |
1057 | 1078 | ||
1058 | printf("\n"); | 1079 | printf("\n"); |
1059 | printf(" %-42s [%s]\n", | 1080 | printf(" %-50s [%s]\n", |
1060 | "rNNN (see 'perf list --help' on how to encode it)", | 1081 | "rNNN (see 'perf list --help' on how to encode it)", |
1061 | event_type_descriptors[PERF_TYPE_RAW]); | 1082 | event_type_descriptors[PERF_TYPE_RAW]); |
1062 | printf("\n"); | 1083 | printf("\n"); |
1063 | 1084 | ||
1064 | printf(" %-42s [%s]\n", | 1085 | printf(" %-50s [%s]\n", |
1065 | "mem:<addr>[:access]", | 1086 | "mem:<addr>[:access]", |
1066 | event_type_descriptors[PERF_TYPE_BREAKPOINT]); | 1087 | event_type_descriptors[PERF_TYPE_BREAKPOINT]); |
1067 | printf("\n"); | 1088 | printf("\n"); |
diff --git a/tools/perf/util/probe-finder.c b/tools/perf/util/probe-finder.c index b7c85ce466a1..3b9d0b800d5c 100644 --- a/tools/perf/util/probe-finder.c +++ b/tools/perf/util/probe-finder.c | |||
@@ -1471,6 +1471,38 @@ static int find_probe_point_by_func(struct probe_finder *pf) | |||
1471 | return _param.retval; | 1471 | return _param.retval; |
1472 | } | 1472 | } |
1473 | 1473 | ||
1474 | struct pubname_callback_param { | ||
1475 | char *function; | ||
1476 | char *file; | ||
1477 | Dwarf_Die *cu_die; | ||
1478 | Dwarf_Die *sp_die; | ||
1479 | int found; | ||
1480 | }; | ||
1481 | |||
1482 | static int pubname_search_cb(Dwarf *dbg, Dwarf_Global *gl, void *data) | ||
1483 | { | ||
1484 | struct pubname_callback_param *param = data; | ||
1485 | |||
1486 | if (dwarf_offdie(dbg, gl->die_offset, param->sp_die)) { | ||
1487 | if (dwarf_tag(param->sp_die) != DW_TAG_subprogram) | ||
1488 | return DWARF_CB_OK; | ||
1489 | |||
1490 | if (die_compare_name(param->sp_die, param->function)) { | ||
1491 | if (!dwarf_offdie(dbg, gl->cu_offset, param->cu_die)) | ||
1492 | return DWARF_CB_OK; | ||
1493 | |||
1494 | if (param->file && | ||
1495 | strtailcmp(param->file, dwarf_decl_file(param->sp_die))) | ||
1496 | return DWARF_CB_OK; | ||
1497 | |||
1498 | param->found = 1; | ||
1499 | return DWARF_CB_ABORT; | ||
1500 | } | ||
1501 | } | ||
1502 | |||
1503 | return DWARF_CB_OK; | ||
1504 | } | ||
1505 | |||
1474 | /* Find probe points from debuginfo */ | 1506 | /* Find probe points from debuginfo */ |
1475 | static int find_probes(int fd, struct probe_finder *pf) | 1507 | static int find_probes(int fd, struct probe_finder *pf) |
1476 | { | 1508 | { |
@@ -1498,6 +1530,28 @@ static int find_probes(int fd, struct probe_finder *pf) | |||
1498 | 1530 | ||
1499 | off = 0; | 1531 | off = 0; |
1500 | line_list__init(&pf->lcache); | 1532 | line_list__init(&pf->lcache); |
1533 | |||
1534 | /* Fastpath: lookup by function name from .debug_pubnames section */ | ||
1535 | if (pp->function) { | ||
1536 | struct pubname_callback_param pubname_param = { | ||
1537 | .function = pp->function, | ||
1538 | .file = pp->file, | ||
1539 | .cu_die = &pf->cu_die, | ||
1540 | .sp_die = &pf->sp_die, | ||
1541 | .found = 0, | ||
1542 | }; | ||
1543 | struct dwarf_callback_param probe_param = { | ||
1544 | .data = pf, | ||
1545 | }; | ||
1546 | |||
1547 | dwarf_getpubnames(dbg, pubname_search_cb, &pubname_param, 0); | ||
1548 | if (pubname_param.found) { | ||
1549 | ret = probe_point_search_cb(&pf->sp_die, &probe_param); | ||
1550 | if (ret) | ||
1551 | goto found; | ||
1552 | } | ||
1553 | } | ||
1554 | |||
1501 | /* Loop on CUs (Compilation Unit) */ | 1555 | /* Loop on CUs (Compilation Unit) */ |
1502 | while (!dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL)) { | 1556 | while (!dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL)) { |
1503 | /* Get the DIE(Debugging Information Entry) of this CU */ | 1557 | /* Get the DIE(Debugging Information Entry) of this CU */ |
@@ -1525,6 +1579,8 @@ static int find_probes(int fd, struct probe_finder *pf) | |||
1525 | } | 1579 | } |
1526 | off = noff; | 1580 | off = noff; |
1527 | } | 1581 | } |
1582 | |||
1583 | found: | ||
1528 | line_list__free(&pf->lcache); | 1584 | line_list__free(&pf->lcache); |
1529 | if (dwfl) | 1585 | if (dwfl) |
1530 | dwfl_end(dwfl); | 1586 | dwfl_end(dwfl); |
@@ -1946,6 +2002,22 @@ int find_line_range(int fd, struct line_range *lr) | |||
1946 | return -EBADF; | 2002 | return -EBADF; |
1947 | } | 2003 | } |
1948 | 2004 | ||
2005 | /* Fastpath: lookup by function name from .debug_pubnames section */ | ||
2006 | if (lr->function) { | ||
2007 | struct pubname_callback_param pubname_param = { | ||
2008 | .function = lr->function, .file = lr->file, | ||
2009 | .cu_die = &lf.cu_die, .sp_die = &lf.sp_die, .found = 0}; | ||
2010 | struct dwarf_callback_param line_range_param = { | ||
2011 | .data = (void *)&lf, .retval = 0}; | ||
2012 | |||
2013 | dwarf_getpubnames(dbg, pubname_search_cb, &pubname_param, 0); | ||
2014 | if (pubname_param.found) { | ||
2015 | line_range_search_cb(&lf.sp_die, &line_range_param); | ||
2016 | if (lf.found) | ||
2017 | goto found; | ||
2018 | } | ||
2019 | } | ||
2020 | |||
1949 | /* Loop on CUs (Compilation Unit) */ | 2021 | /* Loop on CUs (Compilation Unit) */ |
1950 | while (!lf.found && ret >= 0) { | 2022 | while (!lf.found && ret >= 0) { |
1951 | if (dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL) != 0) | 2023 | if (dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL) != 0) |
@@ -1974,6 +2046,7 @@ int find_line_range(int fd, struct line_range *lr) | |||
1974 | off = noff; | 2046 | off = noff; |
1975 | } | 2047 | } |
1976 | 2048 | ||
2049 | found: | ||
1977 | /* Store comp_dir */ | 2050 | /* Store comp_dir */ |
1978 | if (lf.found) { | 2051 | if (lf.found) { |
1979 | comp_dir = cu_get_comp_dir(&lf.cu_die); | 2052 | comp_dir = cu_get_comp_dir(&lf.cu_die); |
diff --git a/tools/perf/util/probe-finder.h b/tools/perf/util/probe-finder.h index beaefc3c1223..605730a366db 100644 --- a/tools/perf/util/probe-finder.h +++ b/tools/perf/util/probe-finder.h | |||
@@ -49,6 +49,7 @@ struct probe_finder { | |||
49 | Dwarf_Addr addr; /* Address */ | 49 | Dwarf_Addr addr; /* Address */ |
50 | const char *fname; /* Real file name */ | 50 | const char *fname; /* Real file name */ |
51 | Dwarf_Die cu_die; /* Current CU */ | 51 | Dwarf_Die cu_die; /* Current CU */ |
52 | Dwarf_Die sp_die; | ||
52 | struct list_head lcache; /* Line cache for lazy match */ | 53 | struct list_head lcache; /* Line cache for lazy match */ |
53 | 54 | ||
54 | /* For variable searching */ | 55 | /* For variable searching */ |
@@ -83,6 +84,7 @@ struct line_finder { | |||
83 | int lno_s; /* Start line number */ | 84 | int lno_s; /* Start line number */ |
84 | int lno_e; /* End line number */ | 85 | int lno_e; /* End line number */ |
85 | Dwarf_Die cu_die; /* Current CU */ | 86 | Dwarf_Die cu_die; /* Current CU */ |
87 | Dwarf_Die sp_die; | ||
86 | int found; | 88 | int found; |
87 | }; | 89 | }; |
88 | 90 | ||
diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c index 99c722672f84..b5c7d818001c 100644 --- a/tools/perf/util/python.c +++ b/tools/perf/util/python.c | |||
@@ -810,6 +810,9 @@ static struct { | |||
810 | { "COUNT_HW_CACHE_RESULT_ACCESS", PERF_COUNT_HW_CACHE_RESULT_ACCESS }, | 810 | { "COUNT_HW_CACHE_RESULT_ACCESS", PERF_COUNT_HW_CACHE_RESULT_ACCESS }, |
811 | { "COUNT_HW_CACHE_RESULT_MISS", PERF_COUNT_HW_CACHE_RESULT_MISS }, | 811 | { "COUNT_HW_CACHE_RESULT_MISS", PERF_COUNT_HW_CACHE_RESULT_MISS }, |
812 | 812 | ||
813 | { "COUNT_HW_STALLED_CYCLES_FRONTEND", PERF_COUNT_HW_STALLED_CYCLES_FRONTEND }, | ||
814 | { "COUNT_HW_STALLED_CYCLES_BACKEND", PERF_COUNT_HW_STALLED_CYCLES_BACKEND }, | ||
815 | |||
813 | { "COUNT_SW_CPU_CLOCK", PERF_COUNT_SW_CPU_CLOCK }, | 816 | { "COUNT_SW_CPU_CLOCK", PERF_COUNT_SW_CPU_CLOCK }, |
814 | { "COUNT_SW_TASK_CLOCK", PERF_COUNT_SW_TASK_CLOCK }, | 817 | { "COUNT_SW_TASK_CLOCK", PERF_COUNT_SW_TASK_CLOCK }, |
815 | { "COUNT_SW_PAGE_FAULTS", PERF_COUNT_SW_PAGE_FAULTS }, | 818 | { "COUNT_SW_PAGE_FAULTS", PERF_COUNT_SW_PAGE_FAULTS }, |
diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index caa224522fea..fff66741f18d 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c | |||
@@ -1156,6 +1156,18 @@ size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp) | |||
1156 | return ret; | 1156 | return ret; |
1157 | } | 1157 | } |
1158 | 1158 | ||
1159 | struct perf_evsel *perf_session__find_first_evtype(struct perf_session *session, | ||
1160 | unsigned int type) | ||
1161 | { | ||
1162 | struct perf_evsel *pos; | ||
1163 | |||
1164 | list_for_each_entry(pos, &session->evlist->entries, node) { | ||
1165 | if (pos->attr.type == type) | ||
1166 | return pos; | ||
1167 | } | ||
1168 | return NULL; | ||
1169 | } | ||
1170 | |||
1159 | void perf_session__print_symbols(union perf_event *event, | 1171 | void perf_session__print_symbols(union perf_event *event, |
1160 | struct perf_sample *sample, | 1172 | struct perf_sample *sample, |
1161 | struct perf_session *session) | 1173 | struct perf_session *session) |
diff --git a/tools/perf/util/session.h b/tools/perf/util/session.h index 1ac481fc1100..8daaa2d15396 100644 --- a/tools/perf/util/session.h +++ b/tools/perf/util/session.h | |||
@@ -162,6 +162,9 @@ static inline int perf_session__parse_sample(struct perf_session *session, | |||
162 | session->sample_id_all, sample); | 162 | session->sample_id_all, sample); |
163 | } | 163 | } |
164 | 164 | ||
165 | struct perf_evsel *perf_session__find_first_evtype(struct perf_session *session, | ||
166 | unsigned int type); | ||
167 | |||
165 | void perf_session__print_symbols(union perf_event *event, | 168 | void perf_session__print_symbols(union perf_event *event, |
166 | struct perf_sample *sample, | 169 | struct perf_sample *sample, |
167 | struct perf_session *session); | 170 | struct perf_session *session); |
diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index f06c10f092ba..516876dfbe52 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c | |||
@@ -31,13 +31,13 @@ | |||
31 | #define NT_GNU_BUILD_ID 3 | 31 | #define NT_GNU_BUILD_ID 3 |
32 | #endif | 32 | #endif |
33 | 33 | ||
34 | static bool dso__build_id_equal(const struct dso *self, u8 *build_id); | 34 | static bool dso__build_id_equal(const struct dso *dso, u8 *build_id); |
35 | static int elf_read_build_id(Elf *elf, void *bf, size_t size); | 35 | static int elf_read_build_id(Elf *elf, void *bf, size_t size); |
36 | static void dsos__add(struct list_head *head, struct dso *dso); | 36 | static void dsos__add(struct list_head *head, struct dso *dso); |
37 | static struct map *map__new2(u64 start, struct dso *dso, enum map_type type); | 37 | static struct map *map__new2(u64 start, struct dso *dso, enum map_type type); |
38 | static int dso__load_kernel_sym(struct dso *self, struct map *map, | 38 | static int dso__load_kernel_sym(struct dso *dso, struct map *map, |
39 | symbol_filter_t filter); | 39 | symbol_filter_t filter); |
40 | static int dso__load_guest_kernel_sym(struct dso *self, struct map *map, | 40 | static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map, |
41 | symbol_filter_t filter); | 41 | symbol_filter_t filter); |
42 | static int vmlinux_path__nr_entries; | 42 | static int vmlinux_path__nr_entries; |
43 | static char **vmlinux_path; | 43 | static char **vmlinux_path; |
@@ -49,27 +49,27 @@ struct symbol_conf symbol_conf = { | |||
49 | .symfs = "", | 49 | .symfs = "", |
50 | }; | 50 | }; |
51 | 51 | ||
52 | int dso__name_len(const struct dso *self) | 52 | int dso__name_len(const struct dso *dso) |
53 | { | 53 | { |
54 | if (verbose) | 54 | if (verbose) |
55 | return self->long_name_len; | 55 | return dso->long_name_len; |
56 | 56 | ||
57 | return self->short_name_len; | 57 | return dso->short_name_len; |
58 | } | 58 | } |
59 | 59 | ||
60 | bool dso__loaded(const struct dso *self, enum map_type type) | 60 | bool dso__loaded(const struct dso *dso, enum map_type type) |
61 | { | 61 | { |
62 | return self->loaded & (1 << type); | 62 | return dso->loaded & (1 << type); |
63 | } | 63 | } |
64 | 64 | ||
65 | bool dso__sorted_by_name(const struct dso *self, enum map_type type) | 65 | bool dso__sorted_by_name(const struct dso *dso, enum map_type type) |
66 | { | 66 | { |
67 | return self->sorted_by_name & (1 << type); | 67 | return dso->sorted_by_name & (1 << type); |
68 | } | 68 | } |
69 | 69 | ||
70 | static void dso__set_sorted_by_name(struct dso *self, enum map_type type) | 70 | static void dso__set_sorted_by_name(struct dso *dso, enum map_type type) |
71 | { | 71 | { |
72 | self->sorted_by_name |= (1 << type); | 72 | dso->sorted_by_name |= (1 << type); |
73 | } | 73 | } |
74 | 74 | ||
75 | bool symbol_type__is_a(char symbol_type, enum map_type map_type) | 75 | bool symbol_type__is_a(char symbol_type, enum map_type map_type) |
@@ -84,9 +84,9 @@ bool symbol_type__is_a(char symbol_type, enum map_type map_type) | |||
84 | } | 84 | } |
85 | } | 85 | } |
86 | 86 | ||
87 | static void symbols__fixup_end(struct rb_root *self) | 87 | static void symbols__fixup_end(struct rb_root *symbols) |
88 | { | 88 | { |
89 | struct rb_node *nd, *prevnd = rb_first(self); | 89 | struct rb_node *nd, *prevnd = rb_first(symbols); |
90 | struct symbol *curr, *prev; | 90 | struct symbol *curr, *prev; |
91 | 91 | ||
92 | if (prevnd == NULL) | 92 | if (prevnd == NULL) |
@@ -107,10 +107,10 @@ static void symbols__fixup_end(struct rb_root *self) | |||
107 | curr->end = roundup(curr->start, 4096); | 107 | curr->end = roundup(curr->start, 4096); |
108 | } | 108 | } |
109 | 109 | ||
110 | static void __map_groups__fixup_end(struct map_groups *self, enum map_type type) | 110 | static void __map_groups__fixup_end(struct map_groups *mg, enum map_type type) |
111 | { | 111 | { |
112 | struct map *prev, *curr; | 112 | struct map *prev, *curr; |
113 | struct rb_node *nd, *prevnd = rb_first(&self->maps[type]); | 113 | struct rb_node *nd, *prevnd = rb_first(&mg->maps[type]); |
114 | 114 | ||
115 | if (prevnd == NULL) | 115 | if (prevnd == NULL) |
116 | return; | 116 | return; |
@@ -130,128 +130,128 @@ static void __map_groups__fixup_end(struct map_groups *self, enum map_type type) | |||
130 | curr->end = ~0ULL; | 130 | curr->end = ~0ULL; |
131 | } | 131 | } |
132 | 132 | ||
133 | static void map_groups__fixup_end(struct map_groups *self) | 133 | static void map_groups__fixup_end(struct map_groups *mg) |
134 | { | 134 | { |
135 | int i; | 135 | int i; |
136 | for (i = 0; i < MAP__NR_TYPES; ++i) | 136 | for (i = 0; i < MAP__NR_TYPES; ++i) |
137 | __map_groups__fixup_end(self, i); | 137 | __map_groups__fixup_end(mg, i); |
138 | } | 138 | } |
139 | 139 | ||
140 | static struct symbol *symbol__new(u64 start, u64 len, u8 binding, | 140 | static struct symbol *symbol__new(u64 start, u64 len, u8 binding, |
141 | const char *name) | 141 | const char *name) |
142 | { | 142 | { |
143 | size_t namelen = strlen(name) + 1; | 143 | size_t namelen = strlen(name) + 1; |
144 | struct symbol *self = calloc(1, (symbol_conf.priv_size + | 144 | struct symbol *sym = calloc(1, (symbol_conf.priv_size + |
145 | sizeof(*self) + namelen)); | 145 | sizeof(*sym) + namelen)); |
146 | if (self == NULL) | 146 | if (sym == NULL) |
147 | return NULL; | 147 | return NULL; |
148 | 148 | ||
149 | if (symbol_conf.priv_size) | 149 | if (symbol_conf.priv_size) |
150 | self = ((void *)self) + symbol_conf.priv_size; | 150 | sym = ((void *)sym) + symbol_conf.priv_size; |
151 | |||
152 | self->start = start; | ||
153 | self->end = len ? start + len - 1 : start; | ||
154 | self->binding = binding; | ||
155 | self->namelen = namelen - 1; | ||
156 | 151 | ||
157 | pr_debug4("%s: %s %#" PRIx64 "-%#" PRIx64 "\n", __func__, name, start, self->end); | 152 | sym->start = start; |
153 | sym->end = len ? start + len - 1 : start; | ||
154 | sym->binding = binding; | ||
155 | sym->namelen = namelen - 1; | ||
158 | 156 | ||
159 | memcpy(self->name, name, namelen); | 157 | pr_debug4("%s: %s %#" PRIx64 "-%#" PRIx64 "\n", |
158 | __func__, name, start, sym->end); | ||
159 | memcpy(sym->name, name, namelen); | ||
160 | 160 | ||
161 | return self; | 161 | return sym; |
162 | } | 162 | } |
163 | 163 | ||
164 | void symbol__delete(struct symbol *self) | 164 | void symbol__delete(struct symbol *sym) |
165 | { | 165 | { |
166 | free(((void *)self) - symbol_conf.priv_size); | 166 | free(((void *)sym) - symbol_conf.priv_size); |
167 | } | 167 | } |
168 | 168 | ||
169 | static size_t symbol__fprintf(struct symbol *self, FILE *fp) | 169 | static size_t symbol__fprintf(struct symbol *sym, FILE *fp) |
170 | { | 170 | { |
171 | return fprintf(fp, " %" PRIx64 "-%" PRIx64 " %c %s\n", | 171 | return fprintf(fp, " %" PRIx64 "-%" PRIx64 " %c %s\n", |
172 | self->start, self->end, | 172 | sym->start, sym->end, |
173 | self->binding == STB_GLOBAL ? 'g' : | 173 | sym->binding == STB_GLOBAL ? 'g' : |
174 | self->binding == STB_LOCAL ? 'l' : 'w', | 174 | sym->binding == STB_LOCAL ? 'l' : 'w', |
175 | self->name); | 175 | sym->name); |
176 | } | 176 | } |
177 | 177 | ||
178 | void dso__set_long_name(struct dso *self, char *name) | 178 | void dso__set_long_name(struct dso *dso, char *name) |
179 | { | 179 | { |
180 | if (name == NULL) | 180 | if (name == NULL) |
181 | return; | 181 | return; |
182 | self->long_name = name; | 182 | dso->long_name = name; |
183 | self->long_name_len = strlen(name); | 183 | dso->long_name_len = strlen(name); |
184 | } | 184 | } |
185 | 185 | ||
186 | static void dso__set_short_name(struct dso *self, const char *name) | 186 | static void dso__set_short_name(struct dso *dso, const char *name) |
187 | { | 187 | { |
188 | if (name == NULL) | 188 | if (name == NULL) |
189 | return; | 189 | return; |
190 | self->short_name = name; | 190 | dso->short_name = name; |
191 | self->short_name_len = strlen(name); | 191 | dso->short_name_len = strlen(name); |
192 | } | 192 | } |
193 | 193 | ||
194 | static void dso__set_basename(struct dso *self) | 194 | static void dso__set_basename(struct dso *dso) |
195 | { | 195 | { |
196 | dso__set_short_name(self, basename(self->long_name)); | 196 | dso__set_short_name(dso, basename(dso->long_name)); |
197 | } | 197 | } |
198 | 198 | ||
199 | struct dso *dso__new(const char *name) | 199 | struct dso *dso__new(const char *name) |
200 | { | 200 | { |
201 | struct dso *self = calloc(1, sizeof(*self) + strlen(name) + 1); | 201 | struct dso *dso = calloc(1, sizeof(*dso) + strlen(name) + 1); |
202 | 202 | ||
203 | if (self != NULL) { | 203 | if (dso != NULL) { |
204 | int i; | 204 | int i; |
205 | strcpy(self->name, name); | 205 | strcpy(dso->name, name); |
206 | dso__set_long_name(self, self->name); | 206 | dso__set_long_name(dso, dso->name); |
207 | dso__set_short_name(self, self->name); | 207 | dso__set_short_name(dso, dso->name); |
208 | for (i = 0; i < MAP__NR_TYPES; ++i) | 208 | for (i = 0; i < MAP__NR_TYPES; ++i) |
209 | self->symbols[i] = self->symbol_names[i] = RB_ROOT; | 209 | dso->symbols[i] = dso->symbol_names[i] = RB_ROOT; |
210 | self->symtab_type = SYMTAB__NOT_FOUND; | 210 | dso->symtab_type = SYMTAB__NOT_FOUND; |
211 | self->loaded = 0; | 211 | dso->loaded = 0; |
212 | self->sorted_by_name = 0; | 212 | dso->sorted_by_name = 0; |
213 | self->has_build_id = 0; | 213 | dso->has_build_id = 0; |
214 | self->kernel = DSO_TYPE_USER; | 214 | dso->kernel = DSO_TYPE_USER; |
215 | INIT_LIST_HEAD(&self->node); | 215 | INIT_LIST_HEAD(&dso->node); |
216 | } | 216 | } |
217 | 217 | ||
218 | return self; | 218 | return dso; |
219 | } | 219 | } |
220 | 220 | ||
221 | static void symbols__delete(struct rb_root *self) | 221 | static void symbols__delete(struct rb_root *symbols) |
222 | { | 222 | { |
223 | struct symbol *pos; | 223 | struct symbol *pos; |
224 | struct rb_node *next = rb_first(self); | 224 | struct rb_node *next = rb_first(symbols); |
225 | 225 | ||
226 | while (next) { | 226 | while (next) { |
227 | pos = rb_entry(next, struct symbol, rb_node); | 227 | pos = rb_entry(next, struct symbol, rb_node); |
228 | next = rb_next(&pos->rb_node); | 228 | next = rb_next(&pos->rb_node); |
229 | rb_erase(&pos->rb_node, self); | 229 | rb_erase(&pos->rb_node, symbols); |
230 | symbol__delete(pos); | 230 | symbol__delete(pos); |
231 | } | 231 | } |
232 | } | 232 | } |
233 | 233 | ||
234 | void dso__delete(struct dso *self) | 234 | void dso__delete(struct dso *dso) |
235 | { | 235 | { |
236 | int i; | 236 | int i; |
237 | for (i = 0; i < MAP__NR_TYPES; ++i) | 237 | for (i = 0; i < MAP__NR_TYPES; ++i) |
238 | symbols__delete(&self->symbols[i]); | 238 | symbols__delete(&dso->symbols[i]); |
239 | if (self->sname_alloc) | 239 | if (dso->sname_alloc) |
240 | free((char *)self->short_name); | 240 | free((char *)dso->short_name); |
241 | if (self->lname_alloc) | 241 | if (dso->lname_alloc) |
242 | free(self->long_name); | 242 | free(dso->long_name); |
243 | free(self); | 243 | free(dso); |
244 | } | 244 | } |
245 | 245 | ||
246 | void dso__set_build_id(struct dso *self, void *build_id) | 246 | void dso__set_build_id(struct dso *dso, void *build_id) |
247 | { | 247 | { |
248 | memcpy(self->build_id, build_id, sizeof(self->build_id)); | 248 | memcpy(dso->build_id, build_id, sizeof(dso->build_id)); |
249 | self->has_build_id = 1; | 249 | dso->has_build_id = 1; |
250 | } | 250 | } |
251 | 251 | ||
252 | static void symbols__insert(struct rb_root *self, struct symbol *sym) | 252 | static void symbols__insert(struct rb_root *symbols, struct symbol *sym) |
253 | { | 253 | { |
254 | struct rb_node **p = &self->rb_node; | 254 | struct rb_node **p = &symbols->rb_node; |
255 | struct rb_node *parent = NULL; | 255 | struct rb_node *parent = NULL; |
256 | const u64 ip = sym->start; | 256 | const u64 ip = sym->start; |
257 | struct symbol *s; | 257 | struct symbol *s; |
@@ -265,17 +265,17 @@ static void symbols__insert(struct rb_root *self, struct symbol *sym) | |||
265 | p = &(*p)->rb_right; | 265 | p = &(*p)->rb_right; |
266 | } | 266 | } |
267 | rb_link_node(&sym->rb_node, parent, p); | 267 | rb_link_node(&sym->rb_node, parent, p); |
268 | rb_insert_color(&sym->rb_node, self); | 268 | rb_insert_color(&sym->rb_node, symbols); |
269 | } | 269 | } |
270 | 270 | ||
271 | static struct symbol *symbols__find(struct rb_root *self, u64 ip) | 271 | static struct symbol *symbols__find(struct rb_root *symbols, u64 ip) |
272 | { | 272 | { |
273 | struct rb_node *n; | 273 | struct rb_node *n; |
274 | 274 | ||
275 | if (self == NULL) | 275 | if (symbols == NULL) |
276 | return NULL; | 276 | return NULL; |
277 | 277 | ||
278 | n = self->rb_node; | 278 | n = symbols->rb_node; |
279 | 279 | ||
280 | while (n) { | 280 | while (n) { |
281 | struct symbol *s = rb_entry(n, struct symbol, rb_node); | 281 | struct symbol *s = rb_entry(n, struct symbol, rb_node); |
@@ -296,9 +296,9 @@ struct symbol_name_rb_node { | |||
296 | struct symbol sym; | 296 | struct symbol sym; |
297 | }; | 297 | }; |
298 | 298 | ||
299 | static void symbols__insert_by_name(struct rb_root *self, struct symbol *sym) | 299 | static void symbols__insert_by_name(struct rb_root *symbols, struct symbol *sym) |
300 | { | 300 | { |
301 | struct rb_node **p = &self->rb_node; | 301 | struct rb_node **p = &symbols->rb_node; |
302 | struct rb_node *parent = NULL; | 302 | struct rb_node *parent = NULL; |
303 | struct symbol_name_rb_node *symn, *s; | 303 | struct symbol_name_rb_node *symn, *s; |
304 | 304 | ||
@@ -313,27 +313,29 @@ static void symbols__insert_by_name(struct rb_root *self, struct symbol *sym) | |||
313 | p = &(*p)->rb_right; | 313 | p = &(*p)->rb_right; |
314 | } | 314 | } |
315 | rb_link_node(&symn->rb_node, parent, p); | 315 | rb_link_node(&symn->rb_node, parent, p); |
316 | rb_insert_color(&symn->rb_node, self); | 316 | rb_insert_color(&symn->rb_node, symbols); |
317 | } | 317 | } |
318 | 318 | ||
319 | static void symbols__sort_by_name(struct rb_root *self, struct rb_root *source) | 319 | static void symbols__sort_by_name(struct rb_root *symbols, |
320 | struct rb_root *source) | ||
320 | { | 321 | { |
321 | struct rb_node *nd; | 322 | struct rb_node *nd; |
322 | 323 | ||
323 | for (nd = rb_first(source); nd; nd = rb_next(nd)) { | 324 | for (nd = rb_first(source); nd; nd = rb_next(nd)) { |
324 | struct symbol *pos = rb_entry(nd, struct symbol, rb_node); | 325 | struct symbol *pos = rb_entry(nd, struct symbol, rb_node); |
325 | symbols__insert_by_name(self, pos); | 326 | symbols__insert_by_name(symbols, pos); |
326 | } | 327 | } |
327 | } | 328 | } |
328 | 329 | ||
329 | static struct symbol *symbols__find_by_name(struct rb_root *self, const char *name) | 330 | static struct symbol *symbols__find_by_name(struct rb_root *symbols, |
331 | const char *name) | ||
330 | { | 332 | { |
331 | struct rb_node *n; | 333 | struct rb_node *n; |
332 | 334 | ||
333 | if (self == NULL) | 335 | if (symbols == NULL) |
334 | return NULL; | 336 | return NULL; |
335 | 337 | ||
336 | n = self->rb_node; | 338 | n = symbols->rb_node; |
337 | 339 | ||
338 | while (n) { | 340 | while (n) { |
339 | struct symbol_name_rb_node *s; | 341 | struct symbol_name_rb_node *s; |
@@ -353,29 +355,29 @@ static struct symbol *symbols__find_by_name(struct rb_root *self, const char *na | |||
353 | return NULL; | 355 | return NULL; |
354 | } | 356 | } |
355 | 357 | ||
356 | struct symbol *dso__find_symbol(struct dso *self, | 358 | struct symbol *dso__find_symbol(struct dso *dso, |
357 | enum map_type type, u64 addr) | 359 | enum map_type type, u64 addr) |
358 | { | 360 | { |
359 | return symbols__find(&self->symbols[type], addr); | 361 | return symbols__find(&dso->symbols[type], addr); |
360 | } | 362 | } |
361 | 363 | ||
362 | struct symbol *dso__find_symbol_by_name(struct dso *self, enum map_type type, | 364 | struct symbol *dso__find_symbol_by_name(struct dso *dso, enum map_type type, |
363 | const char *name) | 365 | const char *name) |
364 | { | 366 | { |
365 | return symbols__find_by_name(&self->symbol_names[type], name); | 367 | return symbols__find_by_name(&dso->symbol_names[type], name); |
366 | } | 368 | } |
367 | 369 | ||
368 | void dso__sort_by_name(struct dso *self, enum map_type type) | 370 | void dso__sort_by_name(struct dso *dso, enum map_type type) |
369 | { | 371 | { |
370 | dso__set_sorted_by_name(self, type); | 372 | dso__set_sorted_by_name(dso, type); |
371 | return symbols__sort_by_name(&self->symbol_names[type], | 373 | return symbols__sort_by_name(&dso->symbol_names[type], |
372 | &self->symbols[type]); | 374 | &dso->symbols[type]); |
373 | } | 375 | } |
374 | 376 | ||
375 | int build_id__sprintf(const u8 *self, int len, char *bf) | 377 | int build_id__sprintf(const u8 *build_id, int len, char *bf) |
376 | { | 378 | { |
377 | char *bid = bf; | 379 | char *bid = bf; |
378 | const u8 *raw = self; | 380 | const u8 *raw = build_id; |
379 | int i; | 381 | int i; |
380 | 382 | ||
381 | for (i = 0; i < len; ++i) { | 383 | for (i = 0; i < len; ++i) { |
@@ -384,24 +386,25 @@ int build_id__sprintf(const u8 *self, int len, char *bf) | |||
384 | bid += 2; | 386 | bid += 2; |
385 | } | 387 | } |
386 | 388 | ||
387 | return raw - self; | 389 | return raw - build_id; |
388 | } | 390 | } |
389 | 391 | ||
390 | size_t dso__fprintf_buildid(struct dso *self, FILE *fp) | 392 | size_t dso__fprintf_buildid(struct dso *dso, FILE *fp) |
391 | { | 393 | { |
392 | char sbuild_id[BUILD_ID_SIZE * 2 + 1]; | 394 | char sbuild_id[BUILD_ID_SIZE * 2 + 1]; |
393 | 395 | ||
394 | build_id__sprintf(self->build_id, sizeof(self->build_id), sbuild_id); | 396 | build_id__sprintf(dso->build_id, sizeof(dso->build_id), sbuild_id); |
395 | return fprintf(fp, "%s", sbuild_id); | 397 | return fprintf(fp, "%s", sbuild_id); |
396 | } | 398 | } |
397 | 399 | ||
398 | size_t dso__fprintf_symbols_by_name(struct dso *self, enum map_type type, FILE *fp) | 400 | size_t dso__fprintf_symbols_by_name(struct dso *dso, |
401 | enum map_type type, FILE *fp) | ||
399 | { | 402 | { |
400 | size_t ret = 0; | 403 | size_t ret = 0; |
401 | struct rb_node *nd; | 404 | struct rb_node *nd; |
402 | struct symbol_name_rb_node *pos; | 405 | struct symbol_name_rb_node *pos; |
403 | 406 | ||
404 | for (nd = rb_first(&self->symbol_names[type]); nd; nd = rb_next(nd)) { | 407 | for (nd = rb_first(&dso->symbol_names[type]); nd; nd = rb_next(nd)) { |
405 | pos = rb_entry(nd, struct symbol_name_rb_node, rb_node); | 408 | pos = rb_entry(nd, struct symbol_name_rb_node, rb_node); |
406 | fprintf(fp, "%s\n", pos->sym.name); | 409 | fprintf(fp, "%s\n", pos->sym.name); |
407 | } | 410 | } |
@@ -409,18 +412,18 @@ size_t dso__fprintf_symbols_by_name(struct dso *self, enum map_type type, FILE * | |||
409 | return ret; | 412 | return ret; |
410 | } | 413 | } |
411 | 414 | ||
412 | size_t dso__fprintf(struct dso *self, enum map_type type, FILE *fp) | 415 | size_t dso__fprintf(struct dso *dso, enum map_type type, FILE *fp) |
413 | { | 416 | { |
414 | struct rb_node *nd; | 417 | struct rb_node *nd; |
415 | size_t ret = fprintf(fp, "dso: %s (", self->short_name); | 418 | size_t ret = fprintf(fp, "dso: %s (", dso->short_name); |
416 | 419 | ||
417 | if (self->short_name != self->long_name) | 420 | if (dso->short_name != dso->long_name) |
418 | ret += fprintf(fp, "%s, ", self->long_name); | 421 | ret += fprintf(fp, "%s, ", dso->long_name); |
419 | ret += fprintf(fp, "%s, %sloaded, ", map_type__name[type], | 422 | ret += fprintf(fp, "%s, %sloaded, ", map_type__name[type], |
420 | self->loaded ? "" : "NOT "); | 423 | dso->loaded ? "" : "NOT "); |
421 | ret += dso__fprintf_buildid(self, fp); | 424 | ret += dso__fprintf_buildid(dso, fp); |
422 | ret += fprintf(fp, ")\n"); | 425 | ret += fprintf(fp, ")\n"); |
423 | for (nd = rb_first(&self->symbols[type]); nd; nd = rb_next(nd)) { | 426 | for (nd = rb_first(&dso->symbols[type]); nd; nd = rb_next(nd)) { |
424 | struct symbol *pos = rb_entry(nd, struct symbol, rb_node); | 427 | struct symbol *pos = rb_entry(nd, struct symbol, rb_node); |
425 | ret += symbol__fprintf(pos, fp); | 428 | ret += symbol__fprintf(pos, fp); |
426 | } | 429 | } |
@@ -543,10 +546,10 @@ static int map__process_kallsym_symbol(void *arg, const char *name, | |||
543 | * so that we can in the next step set the symbol ->end address and then | 546 | * so that we can in the next step set the symbol ->end address and then |
544 | * call kernel_maps__split_kallsyms. | 547 | * call kernel_maps__split_kallsyms. |
545 | */ | 548 | */ |
546 | static int dso__load_all_kallsyms(struct dso *self, const char *filename, | 549 | static int dso__load_all_kallsyms(struct dso *dso, const char *filename, |
547 | struct map *map) | 550 | struct map *map) |
548 | { | 551 | { |
549 | struct process_kallsyms_args args = { .map = map, .dso = self, }; | 552 | struct process_kallsyms_args args = { .map = map, .dso = dso, }; |
550 | return kallsyms__parse(filename, &args, map__process_kallsym_symbol); | 553 | return kallsyms__parse(filename, &args, map__process_kallsym_symbol); |
551 | } | 554 | } |
552 | 555 | ||
@@ -555,7 +558,7 @@ static int dso__load_all_kallsyms(struct dso *self, const char *filename, | |||
555 | * kernel range is broken in several maps, named [kernel].N, as we don't have | 558 | * kernel range is broken in several maps, named [kernel].N, as we don't have |
556 | * the original ELF section names vmlinux have. | 559 | * the original ELF section names vmlinux have. |
557 | */ | 560 | */ |
558 | static int dso__split_kallsyms(struct dso *self, struct map *map, | 561 | static int dso__split_kallsyms(struct dso *dso, struct map *map, |
559 | symbol_filter_t filter) | 562 | symbol_filter_t filter) |
560 | { | 563 | { |
561 | struct map_groups *kmaps = map__kmap(map)->kmaps; | 564 | struct map_groups *kmaps = map__kmap(map)->kmaps; |
@@ -563,7 +566,7 @@ static int dso__split_kallsyms(struct dso *self, struct map *map, | |||
563 | struct map *curr_map = map; | 566 | struct map *curr_map = map; |
564 | struct symbol *pos; | 567 | struct symbol *pos; |
565 | int count = 0, moved = 0; | 568 | int count = 0, moved = 0; |
566 | struct rb_root *root = &self->symbols[map->type]; | 569 | struct rb_root *root = &dso->symbols[map->type]; |
567 | struct rb_node *next = rb_first(root); | 570 | struct rb_node *next = rb_first(root); |
568 | int kernel_range = 0; | 571 | int kernel_range = 0; |
569 | 572 | ||
@@ -582,7 +585,7 @@ static int dso__split_kallsyms(struct dso *self, struct map *map, | |||
582 | 585 | ||
583 | if (strcmp(curr_map->dso->short_name, module)) { | 586 | if (strcmp(curr_map->dso->short_name, module)) { |
584 | if (curr_map != map && | 587 | if (curr_map != map && |
585 | self->kernel == DSO_TYPE_GUEST_KERNEL && | 588 | dso->kernel == DSO_TYPE_GUEST_KERNEL && |
586 | machine__is_default_guest(machine)) { | 589 | machine__is_default_guest(machine)) { |
587 | /* | 590 | /* |
588 | * We assume all symbols of a module are | 591 | * We assume all symbols of a module are |
@@ -618,14 +621,14 @@ static int dso__split_kallsyms(struct dso *self, struct map *map, | |||
618 | pos->end = curr_map->map_ip(curr_map, pos->end); | 621 | pos->end = curr_map->map_ip(curr_map, pos->end); |
619 | } else if (curr_map != map) { | 622 | } else if (curr_map != map) { |
620 | char dso_name[PATH_MAX]; | 623 | char dso_name[PATH_MAX]; |
621 | struct dso *dso; | 624 | struct dso *ndso; |
622 | 625 | ||
623 | if (count == 0) { | 626 | if (count == 0) { |
624 | curr_map = map; | 627 | curr_map = map; |
625 | goto filter_symbol; | 628 | goto filter_symbol; |
626 | } | 629 | } |
627 | 630 | ||
628 | if (self->kernel == DSO_TYPE_GUEST_KERNEL) | 631 | if (dso->kernel == DSO_TYPE_GUEST_KERNEL) |
629 | snprintf(dso_name, sizeof(dso_name), | 632 | snprintf(dso_name, sizeof(dso_name), |
630 | "[guest.kernel].%d", | 633 | "[guest.kernel].%d", |
631 | kernel_range++); | 634 | kernel_range++); |
@@ -634,15 +637,15 @@ static int dso__split_kallsyms(struct dso *self, struct map *map, | |||
634 | "[kernel].%d", | 637 | "[kernel].%d", |
635 | kernel_range++); | 638 | kernel_range++); |
636 | 639 | ||
637 | dso = dso__new(dso_name); | 640 | ndso = dso__new(dso_name); |
638 | if (dso == NULL) | 641 | if (ndso == NULL) |
639 | return -1; | 642 | return -1; |
640 | 643 | ||
641 | dso->kernel = self->kernel; | 644 | ndso->kernel = dso->kernel; |
642 | 645 | ||
643 | curr_map = map__new2(pos->start, dso, map->type); | 646 | curr_map = map__new2(pos->start, ndso, map->type); |
644 | if (curr_map == NULL) { | 647 | if (curr_map == NULL) { |
645 | dso__delete(dso); | 648 | dso__delete(ndso); |
646 | return -1; | 649 | return -1; |
647 | } | 650 | } |
648 | 651 | ||
@@ -665,7 +668,7 @@ discard_symbol: rb_erase(&pos->rb_node, root); | |||
665 | } | 668 | } |
666 | 669 | ||
667 | if (curr_map != map && | 670 | if (curr_map != map && |
668 | self->kernel == DSO_TYPE_GUEST_KERNEL && | 671 | dso->kernel == DSO_TYPE_GUEST_KERNEL && |
669 | machine__is_default_guest(kmaps->machine)) { | 672 | machine__is_default_guest(kmaps->machine)) { |
670 | dso__set_loaded(curr_map->dso, curr_map->type); | 673 | dso__set_loaded(curr_map->dso, curr_map->type); |
671 | } | 674 | } |
@@ -673,21 +676,21 @@ discard_symbol: rb_erase(&pos->rb_node, root); | |||
673 | return count + moved; | 676 | return count + moved; |
674 | } | 677 | } |
675 | 678 | ||
676 | int dso__load_kallsyms(struct dso *self, const char *filename, | 679 | int dso__load_kallsyms(struct dso *dso, const char *filename, |
677 | struct map *map, symbol_filter_t filter) | 680 | struct map *map, symbol_filter_t filter) |
678 | { | 681 | { |
679 | if (dso__load_all_kallsyms(self, filename, map) < 0) | 682 | if (dso__load_all_kallsyms(dso, filename, map) < 0) |
680 | return -1; | 683 | return -1; |
681 | 684 | ||
682 | if (self->kernel == DSO_TYPE_GUEST_KERNEL) | 685 | if (dso->kernel == DSO_TYPE_GUEST_KERNEL) |
683 | self->symtab_type = SYMTAB__GUEST_KALLSYMS; | 686 | dso->symtab_type = SYMTAB__GUEST_KALLSYMS; |
684 | else | 687 | else |
685 | self->symtab_type = SYMTAB__KALLSYMS; | 688 | dso->symtab_type = SYMTAB__KALLSYMS; |
686 | 689 | ||
687 | return dso__split_kallsyms(self, map, filter); | 690 | return dso__split_kallsyms(dso, map, filter); |
688 | } | 691 | } |
689 | 692 | ||
690 | static int dso__load_perf_map(struct dso *self, struct map *map, | 693 | static int dso__load_perf_map(struct dso *dso, struct map *map, |
691 | symbol_filter_t filter) | 694 | symbol_filter_t filter) |
692 | { | 695 | { |
693 | char *line = NULL; | 696 | char *line = NULL; |
@@ -695,7 +698,7 @@ static int dso__load_perf_map(struct dso *self, struct map *map, | |||
695 | FILE *file; | 698 | FILE *file; |
696 | int nr_syms = 0; | 699 | int nr_syms = 0; |
697 | 700 | ||
698 | file = fopen(self->long_name, "r"); | 701 | file = fopen(dso->long_name, "r"); |
699 | if (file == NULL) | 702 | if (file == NULL) |
700 | goto out_failure; | 703 | goto out_failure; |
701 | 704 | ||
@@ -733,7 +736,7 @@ static int dso__load_perf_map(struct dso *self, struct map *map, | |||
733 | if (filter && filter(map, sym)) | 736 | if (filter && filter(map, sym)) |
734 | symbol__delete(sym); | 737 | symbol__delete(sym); |
735 | else { | 738 | else { |
736 | symbols__insert(&self->symbols[map->type], sym); | 739 | symbols__insert(&dso->symbols[map->type], sym); |
737 | nr_syms++; | 740 | nr_syms++; |
738 | } | 741 | } |
739 | } | 742 | } |
@@ -752,7 +755,7 @@ out_failure: | |||
752 | /** | 755 | /** |
753 | * elf_symtab__for_each_symbol - iterate thru all the symbols | 756 | * elf_symtab__for_each_symbol - iterate thru all the symbols |
754 | * | 757 | * |
755 | * @self: struct elf_symtab instance to iterate | 758 | * @syms: struct elf_symtab instance to iterate |
756 | * @idx: uint32_t idx | 759 | * @idx: uint32_t idx |
757 | * @sym: GElf_Sym iterator | 760 | * @sym: GElf_Sym iterator |
758 | */ | 761 | */ |
@@ -852,7 +855,7 @@ static Elf_Scn *elf_section_by_name(Elf *elf, GElf_Ehdr *ep, | |||
852 | * And always look at the original dso, not at debuginfo packages, that | 855 | * And always look at the original dso, not at debuginfo packages, that |
853 | * have the PLT data stripped out (shdr_rel_plt.sh_type == SHT_NOBITS). | 856 | * have the PLT data stripped out (shdr_rel_plt.sh_type == SHT_NOBITS). |
854 | */ | 857 | */ |
855 | static int dso__synthesize_plt_symbols(struct dso *self, struct map *map, | 858 | static int dso__synthesize_plt_symbols(struct dso *dso, struct map *map, |
856 | symbol_filter_t filter) | 859 | symbol_filter_t filter) |
857 | { | 860 | { |
858 | uint32_t nr_rel_entries, idx; | 861 | uint32_t nr_rel_entries, idx; |
@@ -871,7 +874,7 @@ static int dso__synthesize_plt_symbols(struct dso *self, struct map *map, | |||
871 | char name[PATH_MAX]; | 874 | char name[PATH_MAX]; |
872 | 875 | ||
873 | snprintf(name, sizeof(name), "%s%s", | 876 | snprintf(name, sizeof(name), "%s%s", |
874 | symbol_conf.symfs, self->long_name); | 877 | symbol_conf.symfs, dso->long_name); |
875 | fd = open(name, O_RDONLY); | 878 | fd = open(name, O_RDONLY); |
876 | if (fd < 0) | 879 | if (fd < 0) |
877 | goto out; | 880 | goto out; |
@@ -947,7 +950,7 @@ static int dso__synthesize_plt_symbols(struct dso *self, struct map *map, | |||
947 | if (filter && filter(map, f)) | 950 | if (filter && filter(map, f)) |
948 | symbol__delete(f); | 951 | symbol__delete(f); |
949 | else { | 952 | else { |
950 | symbols__insert(&self->symbols[map->type], f); | 953 | symbols__insert(&dso->symbols[map->type], f); |
951 | ++nr; | 954 | ++nr; |
952 | } | 955 | } |
953 | } | 956 | } |
@@ -969,7 +972,7 @@ static int dso__synthesize_plt_symbols(struct dso *self, struct map *map, | |||
969 | if (filter && filter(map, f)) | 972 | if (filter && filter(map, f)) |
970 | symbol__delete(f); | 973 | symbol__delete(f); |
971 | else { | 974 | else { |
972 | symbols__insert(&self->symbols[map->type], f); | 975 | symbols__insert(&dso->symbols[map->type], f); |
973 | ++nr; | 976 | ++nr; |
974 | } | 977 | } |
975 | } | 978 | } |
@@ -985,29 +988,30 @@ out_close: | |||
985 | return nr; | 988 | return nr; |
986 | out: | 989 | out: |
987 | pr_debug("%s: problems reading %s PLT info.\n", | 990 | pr_debug("%s: problems reading %s PLT info.\n", |
988 | __func__, self->long_name); | 991 | __func__, dso->long_name); |
989 | return 0; | 992 | return 0; |
990 | } | 993 | } |
991 | 994 | ||
992 | static bool elf_sym__is_a(GElf_Sym *self, enum map_type type) | 995 | static bool elf_sym__is_a(GElf_Sym *sym, enum map_type type) |
993 | { | 996 | { |
994 | switch (type) { | 997 | switch (type) { |
995 | case MAP__FUNCTION: | 998 | case MAP__FUNCTION: |
996 | return elf_sym__is_function(self); | 999 | return elf_sym__is_function(sym); |
997 | case MAP__VARIABLE: | 1000 | case MAP__VARIABLE: |
998 | return elf_sym__is_object(self); | 1001 | return elf_sym__is_object(sym); |
999 | default: | 1002 | default: |
1000 | return false; | 1003 | return false; |
1001 | } | 1004 | } |
1002 | } | 1005 | } |
1003 | 1006 | ||
1004 | static bool elf_sec__is_a(GElf_Shdr *self, Elf_Data *secstrs, enum map_type type) | 1007 | static bool elf_sec__is_a(GElf_Shdr *shdr, Elf_Data *secstrs, |
1008 | enum map_type type) | ||
1005 | { | 1009 | { |
1006 | switch (type) { | 1010 | switch (type) { |
1007 | case MAP__FUNCTION: | 1011 | case MAP__FUNCTION: |
1008 | return elf_sec__is_text(self, secstrs); | 1012 | return elf_sec__is_text(shdr, secstrs); |
1009 | case MAP__VARIABLE: | 1013 | case MAP__VARIABLE: |
1010 | return elf_sec__is_data(self, secstrs); | 1014 | return elf_sec__is_data(shdr, secstrs); |
1011 | default: | 1015 | default: |
1012 | return false; | 1016 | return false; |
1013 | } | 1017 | } |
@@ -1032,13 +1036,13 @@ static size_t elf_addr_to_index(Elf *elf, GElf_Addr addr) | |||
1032 | return -1; | 1036 | return -1; |
1033 | } | 1037 | } |
1034 | 1038 | ||
1035 | static int dso__load_sym(struct dso *self, struct map *map, const char *name, | 1039 | static int dso__load_sym(struct dso *dso, struct map *map, const char *name, |
1036 | int fd, symbol_filter_t filter, int kmodule, | 1040 | int fd, symbol_filter_t filter, int kmodule, |
1037 | int want_symtab) | 1041 | int want_symtab) |
1038 | { | 1042 | { |
1039 | struct kmap *kmap = self->kernel ? map__kmap(map) : NULL; | 1043 | struct kmap *kmap = dso->kernel ? map__kmap(map) : NULL; |
1040 | struct map *curr_map = map; | 1044 | struct map *curr_map = map; |
1041 | struct dso *curr_dso = self; | 1045 | struct dso *curr_dso = dso; |
1042 | Elf_Data *symstrs, *secstrs; | 1046 | Elf_Data *symstrs, *secstrs; |
1043 | uint32_t nr_syms; | 1047 | uint32_t nr_syms; |
1044 | int err = -1; | 1048 | int err = -1; |
@@ -1064,14 +1068,14 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name, | |||
1064 | } | 1068 | } |
1065 | 1069 | ||
1066 | /* Always reject images with a mismatched build-id: */ | 1070 | /* Always reject images with a mismatched build-id: */ |
1067 | if (self->has_build_id) { | 1071 | if (dso->has_build_id) { |
1068 | u8 build_id[BUILD_ID_SIZE]; | 1072 | u8 build_id[BUILD_ID_SIZE]; |
1069 | 1073 | ||
1070 | if (elf_read_build_id(elf, build_id, | 1074 | if (elf_read_build_id(elf, build_id, |
1071 | BUILD_ID_SIZE) != BUILD_ID_SIZE) | 1075 | BUILD_ID_SIZE) != BUILD_ID_SIZE) |
1072 | goto out_elf_end; | 1076 | goto out_elf_end; |
1073 | 1077 | ||
1074 | if (!dso__build_id_equal(self, build_id)) | 1078 | if (!dso__build_id_equal(dso, build_id)) |
1075 | goto out_elf_end; | 1079 | goto out_elf_end; |
1076 | } | 1080 | } |
1077 | 1081 | ||
@@ -1112,13 +1116,14 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name, | |||
1112 | nr_syms = shdr.sh_size / shdr.sh_entsize; | 1116 | nr_syms = shdr.sh_size / shdr.sh_entsize; |
1113 | 1117 | ||
1114 | memset(&sym, 0, sizeof(sym)); | 1118 | memset(&sym, 0, sizeof(sym)); |
1115 | if (self->kernel == DSO_TYPE_USER) { | 1119 | if (dso->kernel == DSO_TYPE_USER) { |
1116 | self->adjust_symbols = (ehdr.e_type == ET_EXEC || | 1120 | dso->adjust_symbols = (ehdr.e_type == ET_EXEC || |
1117 | elf_section_by_name(elf, &ehdr, &shdr, | 1121 | elf_section_by_name(elf, &ehdr, &shdr, |
1118 | ".gnu.prelink_undo", | 1122 | ".gnu.prelink_undo", |
1119 | NULL) != NULL); | 1123 | NULL) != NULL); |
1120 | } else self->adjust_symbols = 0; | 1124 | } else { |
1121 | 1125 | dso->adjust_symbols = 0; | |
1126 | } | ||
1122 | elf_symtab__for_each_symbol(syms, nr_syms, idx, sym) { | 1127 | elf_symtab__for_each_symbol(syms, nr_syms, idx, sym) { |
1123 | struct symbol *f; | 1128 | struct symbol *f; |
1124 | const char *elf_name = elf_sym__name(&sym, symstrs); | 1129 | const char *elf_name = elf_sym__name(&sym, symstrs); |
@@ -1168,22 +1173,22 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name, | |||
1168 | (sym.st_value & 1)) | 1173 | (sym.st_value & 1)) |
1169 | --sym.st_value; | 1174 | --sym.st_value; |
1170 | 1175 | ||
1171 | if (self->kernel != DSO_TYPE_USER || kmodule) { | 1176 | if (dso->kernel != DSO_TYPE_USER || kmodule) { |
1172 | char dso_name[PATH_MAX]; | 1177 | char dso_name[PATH_MAX]; |
1173 | 1178 | ||
1174 | if (strcmp(section_name, | 1179 | if (strcmp(section_name, |
1175 | (curr_dso->short_name + | 1180 | (curr_dso->short_name + |
1176 | self->short_name_len)) == 0) | 1181 | dso->short_name_len)) == 0) |
1177 | goto new_symbol; | 1182 | goto new_symbol; |
1178 | 1183 | ||
1179 | if (strcmp(section_name, ".text") == 0) { | 1184 | if (strcmp(section_name, ".text") == 0) { |
1180 | curr_map = map; | 1185 | curr_map = map; |
1181 | curr_dso = self; | 1186 | curr_dso = dso; |
1182 | goto new_symbol; | 1187 | goto new_symbol; |
1183 | } | 1188 | } |
1184 | 1189 | ||
1185 | snprintf(dso_name, sizeof(dso_name), | 1190 | snprintf(dso_name, sizeof(dso_name), |
1186 | "%s%s", self->short_name, section_name); | 1191 | "%s%s", dso->short_name, section_name); |
1187 | 1192 | ||
1188 | curr_map = map_groups__find_by_name(kmap->kmaps, map->type, dso_name); | 1193 | curr_map = map_groups__find_by_name(kmap->kmaps, map->type, dso_name); |
1189 | if (curr_map == NULL) { | 1194 | if (curr_map == NULL) { |
@@ -1195,9 +1200,9 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name, | |||
1195 | curr_dso = dso__new(dso_name); | 1200 | curr_dso = dso__new(dso_name); |
1196 | if (curr_dso == NULL) | 1201 | if (curr_dso == NULL) |
1197 | goto out_elf_end; | 1202 | goto out_elf_end; |
1198 | curr_dso->kernel = self->kernel; | 1203 | curr_dso->kernel = dso->kernel; |
1199 | curr_dso->long_name = self->long_name; | 1204 | curr_dso->long_name = dso->long_name; |
1200 | curr_dso->long_name_len = self->long_name_len; | 1205 | curr_dso->long_name_len = dso->long_name_len; |
1201 | curr_map = map__new2(start, curr_dso, | 1206 | curr_map = map__new2(start, curr_dso, |
1202 | map->type); | 1207 | map->type); |
1203 | if (curr_map == NULL) { | 1208 | if (curr_map == NULL) { |
@@ -1206,9 +1211,9 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name, | |||
1206 | } | 1211 | } |
1207 | curr_map->map_ip = identity__map_ip; | 1212 | curr_map->map_ip = identity__map_ip; |
1208 | curr_map->unmap_ip = identity__map_ip; | 1213 | curr_map->unmap_ip = identity__map_ip; |
1209 | curr_dso->symtab_type = self->symtab_type; | 1214 | curr_dso->symtab_type = dso->symtab_type; |
1210 | map_groups__insert(kmap->kmaps, curr_map); | 1215 | map_groups__insert(kmap->kmaps, curr_map); |
1211 | dsos__add(&self->node, curr_dso); | 1216 | dsos__add(&dso->node, curr_dso); |
1212 | dso__set_loaded(curr_dso, map->type); | 1217 | dso__set_loaded(curr_dso, map->type); |
1213 | } else | 1218 | } else |
1214 | curr_dso = curr_map->dso; | 1219 | curr_dso = curr_map->dso; |
@@ -1250,7 +1255,7 @@ new_symbol: | |||
1250 | * For misannotated, zeroed, ASM function sizes. | 1255 | * For misannotated, zeroed, ASM function sizes. |
1251 | */ | 1256 | */ |
1252 | if (nr > 0) { | 1257 | if (nr > 0) { |
1253 | symbols__fixup_end(&self->symbols[map->type]); | 1258 | symbols__fixup_end(&dso->symbols[map->type]); |
1254 | if (kmap) { | 1259 | if (kmap) { |
1255 | /* | 1260 | /* |
1256 | * We need to fixup this here too because we create new | 1261 | * We need to fixup this here too because we create new |
@@ -1266,9 +1271,9 @@ out_close: | |||
1266 | return err; | 1271 | return err; |
1267 | } | 1272 | } |
1268 | 1273 | ||
1269 | static bool dso__build_id_equal(const struct dso *self, u8 *build_id) | 1274 | static bool dso__build_id_equal(const struct dso *dso, u8 *build_id) |
1270 | { | 1275 | { |
1271 | return memcmp(self->build_id, build_id, sizeof(self->build_id)) == 0; | 1276 | return memcmp(dso->build_id, build_id, sizeof(dso->build_id)) == 0; |
1272 | } | 1277 | } |
1273 | 1278 | ||
1274 | bool __dsos__read_build_ids(struct list_head *head, bool with_hits) | 1279 | bool __dsos__read_build_ids(struct list_head *head, bool with_hits) |
@@ -1429,7 +1434,7 @@ out: | |||
1429 | return err; | 1434 | return err; |
1430 | } | 1435 | } |
1431 | 1436 | ||
1432 | char dso__symtab_origin(const struct dso *self) | 1437 | char dso__symtab_origin(const struct dso *dso) |
1433 | { | 1438 | { |
1434 | static const char origin[] = { | 1439 | static const char origin[] = { |
1435 | [SYMTAB__KALLSYMS] = 'k', | 1440 | [SYMTAB__KALLSYMS] = 'k', |
@@ -1444,12 +1449,12 @@ char dso__symtab_origin(const struct dso *self) | |||
1444 | [SYMTAB__GUEST_KMODULE] = 'G', | 1449 | [SYMTAB__GUEST_KMODULE] = 'G', |
1445 | }; | 1450 | }; |
1446 | 1451 | ||
1447 | if (self == NULL || self->symtab_type == SYMTAB__NOT_FOUND) | 1452 | if (dso == NULL || dso->symtab_type == SYMTAB__NOT_FOUND) |
1448 | return '!'; | 1453 | return '!'; |
1449 | return origin[self->symtab_type]; | 1454 | return origin[dso->symtab_type]; |
1450 | } | 1455 | } |
1451 | 1456 | ||
1452 | int dso__load(struct dso *self, struct map *map, symbol_filter_t filter) | 1457 | int dso__load(struct dso *dso, struct map *map, symbol_filter_t filter) |
1453 | { | 1458 | { |
1454 | int size = PATH_MAX; | 1459 | int size = PATH_MAX; |
1455 | char *name; | 1460 | char *name; |
@@ -1459,12 +1464,12 @@ int dso__load(struct dso *self, struct map *map, symbol_filter_t filter) | |||
1459 | const char *root_dir; | 1464 | const char *root_dir; |
1460 | int want_symtab; | 1465 | int want_symtab; |
1461 | 1466 | ||
1462 | dso__set_loaded(self, map->type); | 1467 | dso__set_loaded(dso, map->type); |
1463 | 1468 | ||
1464 | if (self->kernel == DSO_TYPE_KERNEL) | 1469 | if (dso->kernel == DSO_TYPE_KERNEL) |
1465 | return dso__load_kernel_sym(self, map, filter); | 1470 | return dso__load_kernel_sym(dso, map, filter); |
1466 | else if (self->kernel == DSO_TYPE_GUEST_KERNEL) | 1471 | else if (dso->kernel == DSO_TYPE_GUEST_KERNEL) |
1467 | return dso__load_guest_kernel_sym(self, map, filter); | 1472 | return dso__load_guest_kernel_sym(dso, map, filter); |
1468 | 1473 | ||
1469 | if (map->groups && map->groups->machine) | 1474 | if (map->groups && map->groups->machine) |
1470 | machine = map->groups->machine; | 1475 | machine = map->groups->machine; |
@@ -1475,11 +1480,11 @@ int dso__load(struct dso *self, struct map *map, symbol_filter_t filter) | |||
1475 | if (!name) | 1480 | if (!name) |
1476 | return -1; | 1481 | return -1; |
1477 | 1482 | ||
1478 | self->adjust_symbols = 0; | 1483 | dso->adjust_symbols = 0; |
1479 | 1484 | ||
1480 | if (strncmp(self->name, "/tmp/perf-", 10) == 0) { | 1485 | if (strncmp(dso->name, "/tmp/perf-", 10) == 0) { |
1481 | ret = dso__load_perf_map(self, map, filter); | 1486 | ret = dso__load_perf_map(dso, map, filter); |
1482 | self->symtab_type = ret > 0 ? SYMTAB__JAVA_JIT : | 1487 | dso->symtab_type = ret > 0 ? SYMTAB__JAVA_JIT : |
1483 | SYMTAB__NOT_FOUND; | 1488 | SYMTAB__NOT_FOUND; |
1484 | return ret; | 1489 | return ret; |
1485 | } | 1490 | } |
@@ -1490,33 +1495,33 @@ int dso__load(struct dso *self, struct map *map, symbol_filter_t filter) | |||
1490 | */ | 1495 | */ |
1491 | want_symtab = 1; | 1496 | want_symtab = 1; |
1492 | restart: | 1497 | restart: |
1493 | for (self->symtab_type = SYMTAB__BUILD_ID_CACHE; | 1498 | for (dso->symtab_type = SYMTAB__BUILD_ID_CACHE; |
1494 | self->symtab_type != SYMTAB__NOT_FOUND; | 1499 | dso->symtab_type != SYMTAB__NOT_FOUND; |
1495 | self->symtab_type++) { | 1500 | dso->symtab_type++) { |
1496 | switch (self->symtab_type) { | 1501 | switch (dso->symtab_type) { |
1497 | case SYMTAB__BUILD_ID_CACHE: | 1502 | case SYMTAB__BUILD_ID_CACHE: |
1498 | /* skip the locally configured cache if a symfs is given */ | 1503 | /* skip the locally configured cache if a symfs is given */ |
1499 | if (symbol_conf.symfs[0] || | 1504 | if (symbol_conf.symfs[0] || |
1500 | (dso__build_id_filename(self, name, size) == NULL)) { | 1505 | (dso__build_id_filename(dso, name, size) == NULL)) { |
1501 | continue; | 1506 | continue; |
1502 | } | 1507 | } |
1503 | break; | 1508 | break; |
1504 | case SYMTAB__FEDORA_DEBUGINFO: | 1509 | case SYMTAB__FEDORA_DEBUGINFO: |
1505 | snprintf(name, size, "%s/usr/lib/debug%s.debug", | 1510 | snprintf(name, size, "%s/usr/lib/debug%s.debug", |
1506 | symbol_conf.symfs, self->long_name); | 1511 | symbol_conf.symfs, dso->long_name); |
1507 | break; | 1512 | break; |
1508 | case SYMTAB__UBUNTU_DEBUGINFO: | 1513 | case SYMTAB__UBUNTU_DEBUGINFO: |
1509 | snprintf(name, size, "%s/usr/lib/debug%s", | 1514 | snprintf(name, size, "%s/usr/lib/debug%s", |
1510 | symbol_conf.symfs, self->long_name); | 1515 | symbol_conf.symfs, dso->long_name); |
1511 | break; | 1516 | break; |
1512 | case SYMTAB__BUILDID_DEBUGINFO: { | 1517 | case SYMTAB__BUILDID_DEBUGINFO: { |
1513 | char build_id_hex[BUILD_ID_SIZE * 2 + 1]; | 1518 | char build_id_hex[BUILD_ID_SIZE * 2 + 1]; |
1514 | 1519 | ||
1515 | if (!self->has_build_id) | 1520 | if (!dso->has_build_id) |
1516 | continue; | 1521 | continue; |
1517 | 1522 | ||
1518 | build_id__sprintf(self->build_id, | 1523 | build_id__sprintf(dso->build_id, |
1519 | sizeof(self->build_id), | 1524 | sizeof(dso->build_id), |
1520 | build_id_hex); | 1525 | build_id_hex); |
1521 | snprintf(name, size, | 1526 | snprintf(name, size, |
1522 | "%s/usr/lib/debug/.build-id/%.2s/%s.debug", | 1527 | "%s/usr/lib/debug/.build-id/%.2s/%s.debug", |
@@ -1525,7 +1530,7 @@ restart: | |||
1525 | break; | 1530 | break; |
1526 | case SYMTAB__SYSTEM_PATH_DSO: | 1531 | case SYMTAB__SYSTEM_PATH_DSO: |
1527 | snprintf(name, size, "%s%s", | 1532 | snprintf(name, size, "%s%s", |
1528 | symbol_conf.symfs, self->long_name); | 1533 | symbol_conf.symfs, dso->long_name); |
1529 | break; | 1534 | break; |
1530 | case SYMTAB__GUEST_KMODULE: | 1535 | case SYMTAB__GUEST_KMODULE: |
1531 | if (map->groups && machine) | 1536 | if (map->groups && machine) |
@@ -1533,12 +1538,12 @@ restart: | |||
1533 | else | 1538 | else |
1534 | root_dir = ""; | 1539 | root_dir = ""; |
1535 | snprintf(name, size, "%s%s%s", symbol_conf.symfs, | 1540 | snprintf(name, size, "%s%s%s", symbol_conf.symfs, |
1536 | root_dir, self->long_name); | 1541 | root_dir, dso->long_name); |
1537 | break; | 1542 | break; |
1538 | 1543 | ||
1539 | case SYMTAB__SYSTEM_PATH_KMODULE: | 1544 | case SYMTAB__SYSTEM_PATH_KMODULE: |
1540 | snprintf(name, size, "%s%s", symbol_conf.symfs, | 1545 | snprintf(name, size, "%s%s", symbol_conf.symfs, |
1541 | self->long_name); | 1546 | dso->long_name); |
1542 | break; | 1547 | break; |
1543 | default:; | 1548 | default:; |
1544 | } | 1549 | } |
@@ -1548,7 +1553,7 @@ restart: | |||
1548 | if (fd < 0) | 1553 | if (fd < 0) |
1549 | continue; | 1554 | continue; |
1550 | 1555 | ||
1551 | ret = dso__load_sym(self, map, name, fd, filter, 0, | 1556 | ret = dso__load_sym(dso, map, name, fd, filter, 0, |
1552 | want_symtab); | 1557 | want_symtab); |
1553 | close(fd); | 1558 | close(fd); |
1554 | 1559 | ||
@@ -1560,7 +1565,8 @@ restart: | |||
1560 | continue; | 1565 | continue; |
1561 | 1566 | ||
1562 | if (ret > 0) { | 1567 | if (ret > 0) { |
1563 | int nr_plt = dso__synthesize_plt_symbols(self, map, filter); | 1568 | int nr_plt = dso__synthesize_plt_symbols(dso, map, |
1569 | filter); | ||
1564 | if (nr_plt > 0) | 1570 | if (nr_plt > 0) |
1565 | ret += nr_plt; | 1571 | ret += nr_plt; |
1566 | break; | 1572 | break; |
@@ -1577,17 +1583,17 @@ restart: | |||
1577 | } | 1583 | } |
1578 | 1584 | ||
1579 | free(name); | 1585 | free(name); |
1580 | if (ret < 0 && strstr(self->name, " (deleted)") != NULL) | 1586 | if (ret < 0 && strstr(dso->name, " (deleted)") != NULL) |
1581 | return 0; | 1587 | return 0; |
1582 | return ret; | 1588 | return ret; |
1583 | } | 1589 | } |
1584 | 1590 | ||
1585 | struct map *map_groups__find_by_name(struct map_groups *self, | 1591 | struct map *map_groups__find_by_name(struct map_groups *mg, |
1586 | enum map_type type, const char *name) | 1592 | enum map_type type, const char *name) |
1587 | { | 1593 | { |
1588 | struct rb_node *nd; | 1594 | struct rb_node *nd; |
1589 | 1595 | ||
1590 | for (nd = rb_first(&self->maps[type]); nd; nd = rb_next(nd)) { | 1596 | for (nd = rb_first(&mg->maps[type]); nd; nd = rb_next(nd)) { |
1591 | struct map *map = rb_entry(nd, struct map, rb_node); | 1597 | struct map *map = rb_entry(nd, struct map, rb_node); |
1592 | 1598 | ||
1593 | if (map->dso && strcmp(map->dso->short_name, name) == 0) | 1599 | if (map->dso && strcmp(map->dso->short_name, name) == 0) |
@@ -1597,28 +1603,28 @@ struct map *map_groups__find_by_name(struct map_groups *self, | |||
1597 | return NULL; | 1603 | return NULL; |
1598 | } | 1604 | } |
1599 | 1605 | ||
1600 | static int dso__kernel_module_get_build_id(struct dso *self, | 1606 | static int dso__kernel_module_get_build_id(struct dso *dso, |
1601 | const char *root_dir) | 1607 | const char *root_dir) |
1602 | { | 1608 | { |
1603 | char filename[PATH_MAX]; | 1609 | char filename[PATH_MAX]; |
1604 | /* | 1610 | /* |
1605 | * kernel module short names are of the form "[module]" and | 1611 | * kernel module short names are of the form "[module]" and |
1606 | * we need just "module" here. | 1612 | * we need just "module" here. |
1607 | */ | 1613 | */ |
1608 | const char *name = self->short_name + 1; | 1614 | const char *name = dso->short_name + 1; |
1609 | 1615 | ||
1610 | snprintf(filename, sizeof(filename), | 1616 | snprintf(filename, sizeof(filename), |
1611 | "%s/sys/module/%.*s/notes/.note.gnu.build-id", | 1617 | "%s/sys/module/%.*s/notes/.note.gnu.build-id", |
1612 | root_dir, (int)strlen(name) - 1, name); | 1618 | root_dir, (int)strlen(name) - 1, name); |
1613 | 1619 | ||
1614 | if (sysfs__read_build_id(filename, self->build_id, | 1620 | if (sysfs__read_build_id(filename, dso->build_id, |
1615 | sizeof(self->build_id)) == 0) | 1621 | sizeof(dso->build_id)) == 0) |
1616 | self->has_build_id = true; | 1622 | dso->has_build_id = true; |
1617 | 1623 | ||
1618 | return 0; | 1624 | return 0; |
1619 | } | 1625 | } |
1620 | 1626 | ||
1621 | static int map_groups__set_modules_path_dir(struct map_groups *self, | 1627 | static int map_groups__set_modules_path_dir(struct map_groups *mg, |
1622 | const char *dir_name) | 1628 | const char *dir_name) |
1623 | { | 1629 | { |
1624 | struct dirent *dent; | 1630 | struct dirent *dent; |
@@ -1646,7 +1652,7 @@ static int map_groups__set_modules_path_dir(struct map_groups *self, | |||
1646 | 1652 | ||
1647 | snprintf(path, sizeof(path), "%s/%s", | 1653 | snprintf(path, sizeof(path), "%s/%s", |
1648 | dir_name, dent->d_name); | 1654 | dir_name, dent->d_name); |
1649 | ret = map_groups__set_modules_path_dir(self, path); | 1655 | ret = map_groups__set_modules_path_dir(mg, path); |
1650 | if (ret < 0) | 1656 | if (ret < 0) |
1651 | goto out; | 1657 | goto out; |
1652 | } else { | 1658 | } else { |
@@ -1661,7 +1667,8 @@ static int map_groups__set_modules_path_dir(struct map_groups *self, | |||
1661 | (int)(dot - dent->d_name), dent->d_name); | 1667 | (int)(dot - dent->d_name), dent->d_name); |
1662 | 1668 | ||
1663 | strxfrchar(dso_name, '-', '_'); | 1669 | strxfrchar(dso_name, '-', '_'); |
1664 | map = map_groups__find_by_name(self, MAP__FUNCTION, dso_name); | 1670 | map = map_groups__find_by_name(mg, MAP__FUNCTION, |
1671 | dso_name); | ||
1665 | if (map == NULL) | 1672 | if (map == NULL) |
1666 | continue; | 1673 | continue; |
1667 | 1674 | ||
@@ -1711,20 +1718,20 @@ static char *get_kernel_version(const char *root_dir) | |||
1711 | return strdup(name); | 1718 | return strdup(name); |
1712 | } | 1719 | } |
1713 | 1720 | ||
1714 | static int machine__set_modules_path(struct machine *self) | 1721 | static int machine__set_modules_path(struct machine *machine) |
1715 | { | 1722 | { |
1716 | char *version; | 1723 | char *version; |
1717 | char modules_path[PATH_MAX]; | 1724 | char modules_path[PATH_MAX]; |
1718 | 1725 | ||
1719 | version = get_kernel_version(self->root_dir); | 1726 | version = get_kernel_version(machine->root_dir); |
1720 | if (!version) | 1727 | if (!version) |
1721 | return -1; | 1728 | return -1; |
1722 | 1729 | ||
1723 | snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s/kernel", | 1730 | snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s/kernel", |
1724 | self->root_dir, version); | 1731 | machine->root_dir, version); |
1725 | free(version); | 1732 | free(version); |
1726 | 1733 | ||
1727 | return map_groups__set_modules_path_dir(&self->kmaps, modules_path); | 1734 | return map_groups__set_modules_path_dir(&machine->kmaps, modules_path); |
1728 | } | 1735 | } |
1729 | 1736 | ||
1730 | /* | 1737 | /* |
@@ -1734,23 +1741,23 @@ static int machine__set_modules_path(struct machine *self) | |||
1734 | */ | 1741 | */ |
1735 | static struct map *map__new2(u64 start, struct dso *dso, enum map_type type) | 1742 | static struct map *map__new2(u64 start, struct dso *dso, enum map_type type) |
1736 | { | 1743 | { |
1737 | struct map *self = calloc(1, (sizeof(*self) + | 1744 | struct map *map = calloc(1, (sizeof(*map) + |
1738 | (dso->kernel ? sizeof(struct kmap) : 0))); | 1745 | (dso->kernel ? sizeof(struct kmap) : 0))); |
1739 | if (self != NULL) { | 1746 | if (map != NULL) { |
1740 | /* | 1747 | /* |
1741 | * ->end will be filled after we load all the symbols | 1748 | * ->end will be filled after we load all the symbols |
1742 | */ | 1749 | */ |
1743 | map__init(self, type, start, 0, 0, dso); | 1750 | map__init(map, type, start, 0, 0, dso); |
1744 | } | 1751 | } |
1745 | 1752 | ||
1746 | return self; | 1753 | return map; |
1747 | } | 1754 | } |
1748 | 1755 | ||
1749 | struct map *machine__new_module(struct machine *self, u64 start, | 1756 | struct map *machine__new_module(struct machine *machine, u64 start, |
1750 | const char *filename) | 1757 | const char *filename) |
1751 | { | 1758 | { |
1752 | struct map *map; | 1759 | struct map *map; |
1753 | struct dso *dso = __dsos__findnew(&self->kernel_dsos, filename); | 1760 | struct dso *dso = __dsos__findnew(&machine->kernel_dsos, filename); |
1754 | 1761 | ||
1755 | if (dso == NULL) | 1762 | if (dso == NULL) |
1756 | return NULL; | 1763 | return NULL; |
@@ -1759,15 +1766,15 @@ struct map *machine__new_module(struct machine *self, u64 start, | |||
1759 | if (map == NULL) | 1766 | if (map == NULL) |
1760 | return NULL; | 1767 | return NULL; |
1761 | 1768 | ||
1762 | if (machine__is_host(self)) | 1769 | if (machine__is_host(machine)) |
1763 | dso->symtab_type = SYMTAB__SYSTEM_PATH_KMODULE; | 1770 | dso->symtab_type = SYMTAB__SYSTEM_PATH_KMODULE; |
1764 | else | 1771 | else |
1765 | dso->symtab_type = SYMTAB__GUEST_KMODULE; | 1772 | dso->symtab_type = SYMTAB__GUEST_KMODULE; |
1766 | map_groups__insert(&self->kmaps, map); | 1773 | map_groups__insert(&machine->kmaps, map); |
1767 | return map; | 1774 | return map; |
1768 | } | 1775 | } |
1769 | 1776 | ||
1770 | static int machine__create_modules(struct machine *self) | 1777 | static int machine__create_modules(struct machine *machine) |
1771 | { | 1778 | { |
1772 | char *line = NULL; | 1779 | char *line = NULL; |
1773 | size_t n; | 1780 | size_t n; |
@@ -1776,10 +1783,10 @@ static int machine__create_modules(struct machine *self) | |||
1776 | const char *modules; | 1783 | const char *modules; |
1777 | char path[PATH_MAX]; | 1784 | char path[PATH_MAX]; |
1778 | 1785 | ||
1779 | if (machine__is_default_guest(self)) | 1786 | if (machine__is_default_guest(machine)) |
1780 | modules = symbol_conf.default_guest_modules; | 1787 | modules = symbol_conf.default_guest_modules; |
1781 | else { | 1788 | else { |
1782 | sprintf(path, "%s/proc/modules", self->root_dir); | 1789 | sprintf(path, "%s/proc/modules", machine->root_dir); |
1783 | modules = path; | 1790 | modules = path; |
1784 | } | 1791 | } |
1785 | 1792 | ||
@@ -1815,16 +1822,16 @@ static int machine__create_modules(struct machine *self) | |||
1815 | *sep = '\0'; | 1822 | *sep = '\0'; |
1816 | 1823 | ||
1817 | snprintf(name, sizeof(name), "[%s]", line); | 1824 | snprintf(name, sizeof(name), "[%s]", line); |
1818 | map = machine__new_module(self, start, name); | 1825 | map = machine__new_module(machine, start, name); |
1819 | if (map == NULL) | 1826 | if (map == NULL) |
1820 | goto out_delete_line; | 1827 | goto out_delete_line; |
1821 | dso__kernel_module_get_build_id(map->dso, self->root_dir); | 1828 | dso__kernel_module_get_build_id(map->dso, machine->root_dir); |
1822 | } | 1829 | } |
1823 | 1830 | ||
1824 | free(line); | 1831 | free(line); |
1825 | fclose(file); | 1832 | fclose(file); |
1826 | 1833 | ||
1827 | return machine__set_modules_path(self); | 1834 | return machine__set_modules_path(machine); |
1828 | 1835 | ||
1829 | out_delete_line: | 1836 | out_delete_line: |
1830 | free(line); | 1837 | free(line); |
@@ -1832,7 +1839,7 @@ out_failure: | |||
1832 | return -1; | 1839 | return -1; |
1833 | } | 1840 | } |
1834 | 1841 | ||
1835 | int dso__load_vmlinux(struct dso *self, struct map *map, | 1842 | int dso__load_vmlinux(struct dso *dso, struct map *map, |
1836 | const char *vmlinux, symbol_filter_t filter) | 1843 | const char *vmlinux, symbol_filter_t filter) |
1837 | { | 1844 | { |
1838 | int err = -1, fd; | 1845 | int err = -1, fd; |
@@ -1844,9 +1851,9 @@ int dso__load_vmlinux(struct dso *self, struct map *map, | |||
1844 | if (fd < 0) | 1851 | if (fd < 0) |
1845 | return -1; | 1852 | return -1; |
1846 | 1853 | ||
1847 | dso__set_long_name(self, (char *)vmlinux); | 1854 | dso__set_long_name(dso, (char *)vmlinux); |
1848 | dso__set_loaded(self, map->type); | 1855 | dso__set_loaded(dso, map->type); |
1849 | err = dso__load_sym(self, map, symfs_vmlinux, fd, filter, 0, 0); | 1856 | err = dso__load_sym(dso, map, symfs_vmlinux, fd, filter, 0, 0); |
1850 | close(fd); | 1857 | close(fd); |
1851 | 1858 | ||
1852 | if (err > 0) | 1859 | if (err > 0) |
@@ -1855,7 +1862,7 @@ int dso__load_vmlinux(struct dso *self, struct map *map, | |||
1855 | return err; | 1862 | return err; |
1856 | } | 1863 | } |
1857 | 1864 | ||
1858 | int dso__load_vmlinux_path(struct dso *self, struct map *map, | 1865 | int dso__load_vmlinux_path(struct dso *dso, struct map *map, |
1859 | symbol_filter_t filter) | 1866 | symbol_filter_t filter) |
1860 | { | 1867 | { |
1861 | int i, err = 0; | 1868 | int i, err = 0; |
@@ -1864,20 +1871,20 @@ int dso__load_vmlinux_path(struct dso *self, struct map *map, | |||
1864 | pr_debug("Looking at the vmlinux_path (%d entries long)\n", | 1871 | pr_debug("Looking at the vmlinux_path (%d entries long)\n", |
1865 | vmlinux_path__nr_entries + 1); | 1872 | vmlinux_path__nr_entries + 1); |
1866 | 1873 | ||
1867 | filename = dso__build_id_filename(self, NULL, 0); | 1874 | filename = dso__build_id_filename(dso, NULL, 0); |
1868 | if (filename != NULL) { | 1875 | if (filename != NULL) { |
1869 | err = dso__load_vmlinux(self, map, filename, filter); | 1876 | err = dso__load_vmlinux(dso, map, filename, filter); |
1870 | if (err > 0) { | 1877 | if (err > 0) { |
1871 | dso__set_long_name(self, filename); | 1878 | dso__set_long_name(dso, filename); |
1872 | goto out; | 1879 | goto out; |
1873 | } | 1880 | } |
1874 | free(filename); | 1881 | free(filename); |
1875 | } | 1882 | } |
1876 | 1883 | ||
1877 | for (i = 0; i < vmlinux_path__nr_entries; ++i) { | 1884 | for (i = 0; i < vmlinux_path__nr_entries; ++i) { |
1878 | err = dso__load_vmlinux(self, map, vmlinux_path[i], filter); | 1885 | err = dso__load_vmlinux(dso, map, vmlinux_path[i], filter); |
1879 | if (err > 0) { | 1886 | if (err > 0) { |
1880 | dso__set_long_name(self, strdup(vmlinux_path[i])); | 1887 | dso__set_long_name(dso, strdup(vmlinux_path[i])); |
1881 | break; | 1888 | break; |
1882 | } | 1889 | } |
1883 | } | 1890 | } |
@@ -1885,7 +1892,7 @@ out: | |||
1885 | return err; | 1892 | return err; |
1886 | } | 1893 | } |
1887 | 1894 | ||
1888 | static int dso__load_kernel_sym(struct dso *self, struct map *map, | 1895 | static int dso__load_kernel_sym(struct dso *dso, struct map *map, |
1889 | symbol_filter_t filter) | 1896 | symbol_filter_t filter) |
1890 | { | 1897 | { |
1891 | int err; | 1898 | int err; |
@@ -1912,10 +1919,10 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map, | |||
1912 | } | 1919 | } |
1913 | 1920 | ||
1914 | if (symbol_conf.vmlinux_name != NULL) { | 1921 | if (symbol_conf.vmlinux_name != NULL) { |
1915 | err = dso__load_vmlinux(self, map, | 1922 | err = dso__load_vmlinux(dso, map, |
1916 | symbol_conf.vmlinux_name, filter); | 1923 | symbol_conf.vmlinux_name, filter); |
1917 | if (err > 0) { | 1924 | if (err > 0) { |
1918 | dso__set_long_name(self, | 1925 | dso__set_long_name(dso, |
1919 | strdup(symbol_conf.vmlinux_name)); | 1926 | strdup(symbol_conf.vmlinux_name)); |
1920 | goto out_fixup; | 1927 | goto out_fixup; |
1921 | } | 1928 | } |
@@ -1923,7 +1930,7 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map, | |||
1923 | } | 1930 | } |
1924 | 1931 | ||
1925 | if (vmlinux_path != NULL) { | 1932 | if (vmlinux_path != NULL) { |
1926 | err = dso__load_vmlinux_path(self, map, filter); | 1933 | err = dso__load_vmlinux_path(dso, map, filter); |
1927 | if (err > 0) | 1934 | if (err > 0) |
1928 | goto out_fixup; | 1935 | goto out_fixup; |
1929 | } | 1936 | } |
@@ -1937,13 +1944,13 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map, | |||
1937 | * we have a build-id, so check if it is the same as the running kernel, | 1944 | * we have a build-id, so check if it is the same as the running kernel, |
1938 | * using it if it is. | 1945 | * using it if it is. |
1939 | */ | 1946 | */ |
1940 | if (self->has_build_id) { | 1947 | if (dso->has_build_id) { |
1941 | u8 kallsyms_build_id[BUILD_ID_SIZE]; | 1948 | u8 kallsyms_build_id[BUILD_ID_SIZE]; |
1942 | char sbuild_id[BUILD_ID_SIZE * 2 + 1]; | 1949 | char sbuild_id[BUILD_ID_SIZE * 2 + 1]; |
1943 | 1950 | ||
1944 | if (sysfs__read_build_id("/sys/kernel/notes", kallsyms_build_id, | 1951 | if (sysfs__read_build_id("/sys/kernel/notes", kallsyms_build_id, |
1945 | sizeof(kallsyms_build_id)) == 0) { | 1952 | sizeof(kallsyms_build_id)) == 0) { |
1946 | if (dso__build_id_equal(self, kallsyms_build_id)) { | 1953 | if (dso__build_id_equal(dso, kallsyms_build_id)) { |
1947 | kallsyms_filename = "/proc/kallsyms"; | 1954 | kallsyms_filename = "/proc/kallsyms"; |
1948 | goto do_kallsyms; | 1955 | goto do_kallsyms; |
1949 | } | 1956 | } |
@@ -1952,7 +1959,7 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map, | |||
1952 | * Now look if we have it on the build-id cache in | 1959 | * Now look if we have it on the build-id cache in |
1953 | * $HOME/.debug/[kernel.kallsyms]. | 1960 | * $HOME/.debug/[kernel.kallsyms]. |
1954 | */ | 1961 | */ |
1955 | build_id__sprintf(self->build_id, sizeof(self->build_id), | 1962 | build_id__sprintf(dso->build_id, sizeof(dso->build_id), |
1956 | sbuild_id); | 1963 | sbuild_id); |
1957 | 1964 | ||
1958 | if (asprintf(&kallsyms_allocated_filename, | 1965 | if (asprintf(&kallsyms_allocated_filename, |
@@ -1979,7 +1986,7 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map, | |||
1979 | } | 1986 | } |
1980 | 1987 | ||
1981 | do_kallsyms: | 1988 | do_kallsyms: |
1982 | err = dso__load_kallsyms(self, kallsyms_filename, map, filter); | 1989 | err = dso__load_kallsyms(dso, kallsyms_filename, map, filter); |
1983 | if (err > 0) | 1990 | if (err > 0) |
1984 | pr_debug("Using %s for symbols\n", kallsyms_filename); | 1991 | pr_debug("Using %s for symbols\n", kallsyms_filename); |
1985 | free(kallsyms_allocated_filename); | 1992 | free(kallsyms_allocated_filename); |
@@ -1987,7 +1994,7 @@ do_kallsyms: | |||
1987 | if (err > 0) { | 1994 | if (err > 0) { |
1988 | out_fixup: | 1995 | out_fixup: |
1989 | if (kallsyms_filename != NULL) | 1996 | if (kallsyms_filename != NULL) |
1990 | dso__set_long_name(self, strdup("[kernel.kallsyms]")); | 1997 | dso__set_long_name(dso, strdup("[kernel.kallsyms]")); |
1991 | map__fixup_start(map); | 1998 | map__fixup_start(map); |
1992 | map__fixup_end(map); | 1999 | map__fixup_end(map); |
1993 | } | 2000 | } |
@@ -1995,8 +2002,8 @@ out_fixup: | |||
1995 | return err; | 2002 | return err; |
1996 | } | 2003 | } |
1997 | 2004 | ||
1998 | static int dso__load_guest_kernel_sym(struct dso *self, struct map *map, | 2005 | static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map, |
1999 | symbol_filter_t filter) | 2006 | symbol_filter_t filter) |
2000 | { | 2007 | { |
2001 | int err; | 2008 | int err; |
2002 | const char *kallsyms_filename = NULL; | 2009 | const char *kallsyms_filename = NULL; |
@@ -2016,7 +2023,7 @@ static int dso__load_guest_kernel_sym(struct dso *self, struct map *map, | |||
2016 | * Or use file guest_kallsyms inputted by user on commandline | 2023 | * Or use file guest_kallsyms inputted by user on commandline |
2017 | */ | 2024 | */ |
2018 | if (symbol_conf.default_guest_vmlinux_name != NULL) { | 2025 | if (symbol_conf.default_guest_vmlinux_name != NULL) { |
2019 | err = dso__load_vmlinux(self, map, | 2026 | err = dso__load_vmlinux(dso, map, |
2020 | symbol_conf.default_guest_vmlinux_name, filter); | 2027 | symbol_conf.default_guest_vmlinux_name, filter); |
2021 | goto out_try_fixup; | 2028 | goto out_try_fixup; |
2022 | } | 2029 | } |
@@ -2029,7 +2036,7 @@ static int dso__load_guest_kernel_sym(struct dso *self, struct map *map, | |||
2029 | kallsyms_filename = path; | 2036 | kallsyms_filename = path; |
2030 | } | 2037 | } |
2031 | 2038 | ||
2032 | err = dso__load_kallsyms(self, kallsyms_filename, map, filter); | 2039 | err = dso__load_kallsyms(dso, kallsyms_filename, map, filter); |
2033 | if (err > 0) | 2040 | if (err > 0) |
2034 | pr_debug("Using %s for symbols\n", kallsyms_filename); | 2041 | pr_debug("Using %s for symbols\n", kallsyms_filename); |
2035 | 2042 | ||
@@ -2037,7 +2044,7 @@ out_try_fixup: | |||
2037 | if (err > 0) { | 2044 | if (err > 0) { |
2038 | if (kallsyms_filename != NULL) { | 2045 | if (kallsyms_filename != NULL) { |
2039 | machine__mmap_name(machine, path, sizeof(path)); | 2046 | machine__mmap_name(machine, path, sizeof(path)); |
2040 | dso__set_long_name(self, strdup(path)); | 2047 | dso__set_long_name(dso, strdup(path)); |
2041 | } | 2048 | } |
2042 | map__fixup_start(map); | 2049 | map__fixup_start(map); |
2043 | map__fixup_end(map); | 2050 | map__fixup_end(map); |
@@ -2090,12 +2097,12 @@ size_t __dsos__fprintf(struct list_head *head, FILE *fp) | |||
2090 | return ret; | 2097 | return ret; |
2091 | } | 2098 | } |
2092 | 2099 | ||
2093 | size_t machines__fprintf_dsos(struct rb_root *self, FILE *fp) | 2100 | size_t machines__fprintf_dsos(struct rb_root *machines, FILE *fp) |
2094 | { | 2101 | { |
2095 | struct rb_node *nd; | 2102 | struct rb_node *nd; |
2096 | size_t ret = 0; | 2103 | size_t ret = 0; |
2097 | 2104 | ||
2098 | for (nd = rb_first(self); nd; nd = rb_next(nd)) { | 2105 | for (nd = rb_first(machines); nd; nd = rb_next(nd)) { |
2099 | struct machine *pos = rb_entry(nd, struct machine, rb_node); | 2106 | struct machine *pos = rb_entry(nd, struct machine, rb_node); |
2100 | ret += __dsos__fprintf(&pos->kernel_dsos, fp); | 2107 | ret += __dsos__fprintf(&pos->kernel_dsos, fp); |
2101 | ret += __dsos__fprintf(&pos->user_dsos, fp); | 2108 | ret += __dsos__fprintf(&pos->user_dsos, fp); |
@@ -2119,18 +2126,20 @@ static size_t __dsos__fprintf_buildid(struct list_head *head, FILE *fp, | |||
2119 | return ret; | 2126 | return ret; |
2120 | } | 2127 | } |
2121 | 2128 | ||
2122 | size_t machine__fprintf_dsos_buildid(struct machine *self, FILE *fp, bool with_hits) | 2129 | size_t machine__fprintf_dsos_buildid(struct machine *machine, FILE *fp, |
2130 | bool with_hits) | ||
2123 | { | 2131 | { |
2124 | return __dsos__fprintf_buildid(&self->kernel_dsos, fp, with_hits) + | 2132 | return __dsos__fprintf_buildid(&machine->kernel_dsos, fp, with_hits) + |
2125 | __dsos__fprintf_buildid(&self->user_dsos, fp, with_hits); | 2133 | __dsos__fprintf_buildid(&machine->user_dsos, fp, with_hits); |
2126 | } | 2134 | } |
2127 | 2135 | ||
2128 | size_t machines__fprintf_dsos_buildid(struct rb_root *self, FILE *fp, bool with_hits) | 2136 | size_t machines__fprintf_dsos_buildid(struct rb_root *machines, |
2137 | FILE *fp, bool with_hits) | ||
2129 | { | 2138 | { |
2130 | struct rb_node *nd; | 2139 | struct rb_node *nd; |
2131 | size_t ret = 0; | 2140 | size_t ret = 0; |
2132 | 2141 | ||
2133 | for (nd = rb_first(self); nd; nd = rb_next(nd)) { | 2142 | for (nd = rb_first(machines); nd; nd = rb_next(nd)) { |
2134 | struct machine *pos = rb_entry(nd, struct machine, rb_node); | 2143 | struct machine *pos = rb_entry(nd, struct machine, rb_node); |
2135 | ret += machine__fprintf_dsos_buildid(pos, fp, with_hits); | 2144 | ret += machine__fprintf_dsos_buildid(pos, fp, with_hits); |
2136 | } | 2145 | } |
@@ -2139,59 +2148,59 @@ size_t machines__fprintf_dsos_buildid(struct rb_root *self, FILE *fp, bool with_ | |||
2139 | 2148 | ||
2140 | struct dso *dso__new_kernel(const char *name) | 2149 | struct dso *dso__new_kernel(const char *name) |
2141 | { | 2150 | { |
2142 | struct dso *self = dso__new(name ?: "[kernel.kallsyms]"); | 2151 | struct dso *dso = dso__new(name ?: "[kernel.kallsyms]"); |
2143 | 2152 | ||
2144 | if (self != NULL) { | 2153 | if (dso != NULL) { |
2145 | dso__set_short_name(self, "[kernel]"); | 2154 | dso__set_short_name(dso, "[kernel]"); |
2146 | self->kernel = DSO_TYPE_KERNEL; | 2155 | dso->kernel = DSO_TYPE_KERNEL; |
2147 | } | 2156 | } |
2148 | 2157 | ||
2149 | return self; | 2158 | return dso; |
2150 | } | 2159 | } |
2151 | 2160 | ||
2152 | static struct dso *dso__new_guest_kernel(struct machine *machine, | 2161 | static struct dso *dso__new_guest_kernel(struct machine *machine, |
2153 | const char *name) | 2162 | const char *name) |
2154 | { | 2163 | { |
2155 | char bf[PATH_MAX]; | 2164 | char bf[PATH_MAX]; |
2156 | struct dso *self = dso__new(name ?: machine__mmap_name(machine, bf, sizeof(bf))); | 2165 | struct dso *dso = dso__new(name ?: machine__mmap_name(machine, bf, |
2157 | 2166 | sizeof(bf))); | |
2158 | if (self != NULL) { | 2167 | if (dso != NULL) { |
2159 | dso__set_short_name(self, "[guest.kernel]"); | 2168 | dso__set_short_name(dso, "[guest.kernel]"); |
2160 | self->kernel = DSO_TYPE_GUEST_KERNEL; | 2169 | dso->kernel = DSO_TYPE_GUEST_KERNEL; |
2161 | } | 2170 | } |
2162 | 2171 | ||
2163 | return self; | 2172 | return dso; |
2164 | } | 2173 | } |
2165 | 2174 | ||
2166 | void dso__read_running_kernel_build_id(struct dso *self, struct machine *machine) | 2175 | void dso__read_running_kernel_build_id(struct dso *dso, struct machine *machine) |
2167 | { | 2176 | { |
2168 | char path[PATH_MAX]; | 2177 | char path[PATH_MAX]; |
2169 | 2178 | ||
2170 | if (machine__is_default_guest(machine)) | 2179 | if (machine__is_default_guest(machine)) |
2171 | return; | 2180 | return; |
2172 | sprintf(path, "%s/sys/kernel/notes", machine->root_dir); | 2181 | sprintf(path, "%s/sys/kernel/notes", machine->root_dir); |
2173 | if (sysfs__read_build_id(path, self->build_id, | 2182 | if (sysfs__read_build_id(path, dso->build_id, |
2174 | sizeof(self->build_id)) == 0) | 2183 | sizeof(dso->build_id)) == 0) |
2175 | self->has_build_id = true; | 2184 | dso->has_build_id = true; |
2176 | } | 2185 | } |
2177 | 2186 | ||
2178 | static struct dso *machine__create_kernel(struct machine *self) | 2187 | static struct dso *machine__create_kernel(struct machine *machine) |
2179 | { | 2188 | { |
2180 | const char *vmlinux_name = NULL; | 2189 | const char *vmlinux_name = NULL; |
2181 | struct dso *kernel; | 2190 | struct dso *kernel; |
2182 | 2191 | ||
2183 | if (machine__is_host(self)) { | 2192 | if (machine__is_host(machine)) { |
2184 | vmlinux_name = symbol_conf.vmlinux_name; | 2193 | vmlinux_name = symbol_conf.vmlinux_name; |
2185 | kernel = dso__new_kernel(vmlinux_name); | 2194 | kernel = dso__new_kernel(vmlinux_name); |
2186 | } else { | 2195 | } else { |
2187 | if (machine__is_default_guest(self)) | 2196 | if (machine__is_default_guest(machine)) |
2188 | vmlinux_name = symbol_conf.default_guest_vmlinux_name; | 2197 | vmlinux_name = symbol_conf.default_guest_vmlinux_name; |
2189 | kernel = dso__new_guest_kernel(self, vmlinux_name); | 2198 | kernel = dso__new_guest_kernel(machine, vmlinux_name); |
2190 | } | 2199 | } |
2191 | 2200 | ||
2192 | if (kernel != NULL) { | 2201 | if (kernel != NULL) { |
2193 | dso__read_running_kernel_build_id(kernel, self); | 2202 | dso__read_running_kernel_build_id(kernel, machine); |
2194 | dsos__add(&self->kernel_dsos, kernel); | 2203 | dsos__add(&machine->kernel_dsos, kernel); |
2195 | } | 2204 | } |
2196 | return kernel; | 2205 | return kernel; |
2197 | } | 2206 | } |
@@ -2236,41 +2245,43 @@ static u64 machine__get_kernel_start_addr(struct machine *machine) | |||
2236 | return args.start; | 2245 | return args.start; |
2237 | } | 2246 | } |
2238 | 2247 | ||
2239 | int __machine__create_kernel_maps(struct machine *self, struct dso *kernel) | 2248 | int __machine__create_kernel_maps(struct machine *machine, struct dso *kernel) |
2240 | { | 2249 | { |
2241 | enum map_type type; | 2250 | enum map_type type; |
2242 | u64 start = machine__get_kernel_start_addr(self); | 2251 | u64 start = machine__get_kernel_start_addr(machine); |
2243 | 2252 | ||
2244 | for (type = 0; type < MAP__NR_TYPES; ++type) { | 2253 | for (type = 0; type < MAP__NR_TYPES; ++type) { |
2245 | struct kmap *kmap; | 2254 | struct kmap *kmap; |
2246 | 2255 | ||
2247 | self->vmlinux_maps[type] = map__new2(start, kernel, type); | 2256 | machine->vmlinux_maps[type] = map__new2(start, kernel, type); |
2248 | if (self->vmlinux_maps[type] == NULL) | 2257 | if (machine->vmlinux_maps[type] == NULL) |
2249 | return -1; | 2258 | return -1; |
2250 | 2259 | ||
2251 | self->vmlinux_maps[type]->map_ip = | 2260 | machine->vmlinux_maps[type]->map_ip = |
2252 | self->vmlinux_maps[type]->unmap_ip = identity__map_ip; | 2261 | machine->vmlinux_maps[type]->unmap_ip = |
2253 | 2262 | identity__map_ip; | |
2254 | kmap = map__kmap(self->vmlinux_maps[type]); | 2263 | kmap = map__kmap(machine->vmlinux_maps[type]); |
2255 | kmap->kmaps = &self->kmaps; | 2264 | kmap->kmaps = &machine->kmaps; |
2256 | map_groups__insert(&self->kmaps, self->vmlinux_maps[type]); | 2265 | map_groups__insert(&machine->kmaps, |
2266 | machine->vmlinux_maps[type]); | ||
2257 | } | 2267 | } |
2258 | 2268 | ||
2259 | return 0; | 2269 | return 0; |
2260 | } | 2270 | } |
2261 | 2271 | ||
2262 | void machine__destroy_kernel_maps(struct machine *self) | 2272 | void machine__destroy_kernel_maps(struct machine *machine) |
2263 | { | 2273 | { |
2264 | enum map_type type; | 2274 | enum map_type type; |
2265 | 2275 | ||
2266 | for (type = 0; type < MAP__NR_TYPES; ++type) { | 2276 | for (type = 0; type < MAP__NR_TYPES; ++type) { |
2267 | struct kmap *kmap; | 2277 | struct kmap *kmap; |
2268 | 2278 | ||
2269 | if (self->vmlinux_maps[type] == NULL) | 2279 | if (machine->vmlinux_maps[type] == NULL) |
2270 | continue; | 2280 | continue; |
2271 | 2281 | ||
2272 | kmap = map__kmap(self->vmlinux_maps[type]); | 2282 | kmap = map__kmap(machine->vmlinux_maps[type]); |
2273 | map_groups__remove(&self->kmaps, self->vmlinux_maps[type]); | 2283 | map_groups__remove(&machine->kmaps, |
2284 | machine->vmlinux_maps[type]); | ||
2274 | if (kmap->ref_reloc_sym) { | 2285 | if (kmap->ref_reloc_sym) { |
2275 | /* | 2286 | /* |
2276 | * ref_reloc_sym is shared among all maps, so free just | 2287 | * ref_reloc_sym is shared among all maps, so free just |
@@ -2284,25 +2295,25 @@ void machine__destroy_kernel_maps(struct machine *self) | |||
2284 | kmap->ref_reloc_sym = NULL; | 2295 | kmap->ref_reloc_sym = NULL; |
2285 | } | 2296 | } |
2286 | 2297 | ||
2287 | map__delete(self->vmlinux_maps[type]); | 2298 | map__delete(machine->vmlinux_maps[type]); |
2288 | self->vmlinux_maps[type] = NULL; | 2299 | machine->vmlinux_maps[type] = NULL; |
2289 | } | 2300 | } |
2290 | } | 2301 | } |
2291 | 2302 | ||
2292 | int machine__create_kernel_maps(struct machine *self) | 2303 | int machine__create_kernel_maps(struct machine *machine) |
2293 | { | 2304 | { |
2294 | struct dso *kernel = machine__create_kernel(self); | 2305 | struct dso *kernel = machine__create_kernel(machine); |
2295 | 2306 | ||
2296 | if (kernel == NULL || | 2307 | if (kernel == NULL || |
2297 | __machine__create_kernel_maps(self, kernel) < 0) | 2308 | __machine__create_kernel_maps(machine, kernel) < 0) |
2298 | return -1; | 2309 | return -1; |
2299 | 2310 | ||
2300 | if (symbol_conf.use_modules && machine__create_modules(self) < 0) | 2311 | if (symbol_conf.use_modules && machine__create_modules(machine) < 0) |
2301 | pr_debug("Problems creating module maps, continuing anyway...\n"); | 2312 | pr_debug("Problems creating module maps, continuing anyway...\n"); |
2302 | /* | 2313 | /* |
2303 | * Now that we have all the maps created, just set the ->end of them: | 2314 | * Now that we have all the maps created, just set the ->end of them: |
2304 | */ | 2315 | */ |
2305 | map_groups__fixup_end(&self->kmaps); | 2316 | map_groups__fixup_end(&machine->kmaps); |
2306 | return 0; | 2317 | return 0; |
2307 | } | 2318 | } |
2308 | 2319 | ||
@@ -2366,11 +2377,11 @@ out_fail: | |||
2366 | return -1; | 2377 | return -1; |
2367 | } | 2378 | } |
2368 | 2379 | ||
2369 | size_t machine__fprintf_vmlinux_path(struct machine *self, FILE *fp) | 2380 | size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp) |
2370 | { | 2381 | { |
2371 | int i; | 2382 | int i; |
2372 | size_t printed = 0; | 2383 | size_t printed = 0; |
2373 | struct dso *kdso = self->vmlinux_maps[MAP__FUNCTION]->dso; | 2384 | struct dso *kdso = machine->vmlinux_maps[MAP__FUNCTION]->dso; |
2374 | 2385 | ||
2375 | if (kdso->has_build_id) { | 2386 | if (kdso->has_build_id) { |
2376 | char filename[PATH_MAX]; | 2387 | char filename[PATH_MAX]; |
@@ -2467,9 +2478,9 @@ void symbol__exit(void) | |||
2467 | symbol_conf.initialized = false; | 2478 | symbol_conf.initialized = false; |
2468 | } | 2479 | } |
2469 | 2480 | ||
2470 | int machines__create_kernel_maps(struct rb_root *self, pid_t pid) | 2481 | int machines__create_kernel_maps(struct rb_root *machines, pid_t pid) |
2471 | { | 2482 | { |
2472 | struct machine *machine = machines__findnew(self, pid); | 2483 | struct machine *machine = machines__findnew(machines, pid); |
2473 | 2484 | ||
2474 | if (machine == NULL) | 2485 | if (machine == NULL) |
2475 | return -1; | 2486 | return -1; |
@@ -2520,7 +2531,7 @@ char *strxfrchar(char *s, char from, char to) | |||
2520 | return s; | 2531 | return s; |
2521 | } | 2532 | } |
2522 | 2533 | ||
2523 | int machines__create_guest_kernel_maps(struct rb_root *self) | 2534 | int machines__create_guest_kernel_maps(struct rb_root *machines) |
2524 | { | 2535 | { |
2525 | int ret = 0; | 2536 | int ret = 0; |
2526 | struct dirent **namelist = NULL; | 2537 | struct dirent **namelist = NULL; |
@@ -2531,7 +2542,7 @@ int machines__create_guest_kernel_maps(struct rb_root *self) | |||
2531 | if (symbol_conf.default_guest_vmlinux_name || | 2542 | if (symbol_conf.default_guest_vmlinux_name || |
2532 | symbol_conf.default_guest_modules || | 2543 | symbol_conf.default_guest_modules || |
2533 | symbol_conf.default_guest_kallsyms) { | 2544 | symbol_conf.default_guest_kallsyms) { |
2534 | machines__create_kernel_maps(self, DEFAULT_GUEST_KERNEL_ID); | 2545 | machines__create_kernel_maps(machines, DEFAULT_GUEST_KERNEL_ID); |
2535 | } | 2546 | } |
2536 | 2547 | ||
2537 | if (symbol_conf.guestmount) { | 2548 | if (symbol_conf.guestmount) { |
@@ -2552,7 +2563,7 @@ int machines__create_guest_kernel_maps(struct rb_root *self) | |||
2552 | pr_debug("Can't access file %s\n", path); | 2563 | pr_debug("Can't access file %s\n", path); |
2553 | goto failure; | 2564 | goto failure; |
2554 | } | 2565 | } |
2555 | machines__create_kernel_maps(self, pid); | 2566 | machines__create_kernel_maps(machines, pid); |
2556 | } | 2567 | } |
2557 | failure: | 2568 | failure: |
2558 | free(namelist); | 2569 | free(namelist); |
@@ -2561,23 +2572,23 @@ failure: | |||
2561 | return ret; | 2572 | return ret; |
2562 | } | 2573 | } |
2563 | 2574 | ||
2564 | void machines__destroy_guest_kernel_maps(struct rb_root *self) | 2575 | void machines__destroy_guest_kernel_maps(struct rb_root *machines) |
2565 | { | 2576 | { |
2566 | struct rb_node *next = rb_first(self); | 2577 | struct rb_node *next = rb_first(machines); |
2567 | 2578 | ||
2568 | while (next) { | 2579 | while (next) { |
2569 | struct machine *pos = rb_entry(next, struct machine, rb_node); | 2580 | struct machine *pos = rb_entry(next, struct machine, rb_node); |
2570 | 2581 | ||
2571 | next = rb_next(&pos->rb_node); | 2582 | next = rb_next(&pos->rb_node); |
2572 | rb_erase(&pos->rb_node, self); | 2583 | rb_erase(&pos->rb_node, machines); |
2573 | machine__delete(pos); | 2584 | machine__delete(pos); |
2574 | } | 2585 | } |
2575 | } | 2586 | } |
2576 | 2587 | ||
2577 | int machine__load_kallsyms(struct machine *self, const char *filename, | 2588 | int machine__load_kallsyms(struct machine *machine, const char *filename, |
2578 | enum map_type type, symbol_filter_t filter) | 2589 | enum map_type type, symbol_filter_t filter) |
2579 | { | 2590 | { |
2580 | struct map *map = self->vmlinux_maps[type]; | 2591 | struct map *map = machine->vmlinux_maps[type]; |
2581 | int ret = dso__load_kallsyms(map->dso, filename, map, filter); | 2592 | int ret = dso__load_kallsyms(map->dso, filename, map, filter); |
2582 | 2593 | ||
2583 | if (ret > 0) { | 2594 | if (ret > 0) { |
@@ -2587,16 +2598,16 @@ int machine__load_kallsyms(struct machine *self, const char *filename, | |||
2587 | * kernel, with modules between them, fixup the end of all | 2598 | * kernel, with modules between them, fixup the end of all |
2588 | * sections. | 2599 | * sections. |
2589 | */ | 2600 | */ |
2590 | __map_groups__fixup_end(&self->kmaps, type); | 2601 | __map_groups__fixup_end(&machine->kmaps, type); |
2591 | } | 2602 | } |
2592 | 2603 | ||
2593 | return ret; | 2604 | return ret; |
2594 | } | 2605 | } |
2595 | 2606 | ||
2596 | int machine__load_vmlinux_path(struct machine *self, enum map_type type, | 2607 | int machine__load_vmlinux_path(struct machine *machine, enum map_type type, |
2597 | symbol_filter_t filter) | 2608 | symbol_filter_t filter) |
2598 | { | 2609 | { |
2599 | struct map *map = self->vmlinux_maps[type]; | 2610 | struct map *map = machine->vmlinux_maps[type]; |
2600 | int ret = dso__load_vmlinux_path(map->dso, map, filter); | 2611 | int ret = dso__load_vmlinux_path(map->dso, map, filter); |
2601 | 2612 | ||
2602 | if (ret > 0) { | 2613 | if (ret > 0) { |
diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h index 713b0b40cc4a..242de0101a86 100644 --- a/tools/perf/util/symbol.h +++ b/tools/perf/util/symbol.h | |||
@@ -62,7 +62,7 @@ struct symbol { | |||
62 | char name[0]; | 62 | char name[0]; |
63 | }; | 63 | }; |
64 | 64 | ||
65 | void symbol__delete(struct symbol *self); | 65 | void symbol__delete(struct symbol *sym); |
66 | 66 | ||
67 | struct strlist; | 67 | struct strlist; |
68 | 68 | ||
@@ -96,9 +96,9 @@ struct symbol_conf { | |||
96 | 96 | ||
97 | extern struct symbol_conf symbol_conf; | 97 | extern struct symbol_conf symbol_conf; |
98 | 98 | ||
99 | static inline void *symbol__priv(struct symbol *self) | 99 | static inline void *symbol__priv(struct symbol *sym) |
100 | { | 100 | { |
101 | return ((void *)self) - symbol_conf.priv_size; | 101 | return ((void *)sym) - symbol_conf.priv_size; |
102 | } | 102 | } |
103 | 103 | ||
104 | struct ref_reloc_sym { | 104 | struct ref_reloc_sym { |
@@ -155,43 +155,45 @@ struct dso { | |||
155 | 155 | ||
156 | struct dso *dso__new(const char *name); | 156 | struct dso *dso__new(const char *name); |
157 | struct dso *dso__new_kernel(const char *name); | 157 | struct dso *dso__new_kernel(const char *name); |
158 | void dso__delete(struct dso *self); | 158 | void dso__delete(struct dso *dso); |
159 | 159 | ||
160 | int dso__name_len(const struct dso *self); | 160 | int dso__name_len(const struct dso *dso); |
161 | 161 | ||
162 | bool dso__loaded(const struct dso *self, enum map_type type); | 162 | bool dso__loaded(const struct dso *dso, enum map_type type); |
163 | bool dso__sorted_by_name(const struct dso *self, enum map_type type); | 163 | bool dso__sorted_by_name(const struct dso *dso, enum map_type type); |
164 | 164 | ||
165 | static inline void dso__set_loaded(struct dso *self, enum map_type type) | 165 | static inline void dso__set_loaded(struct dso *dso, enum map_type type) |
166 | { | 166 | { |
167 | self->loaded |= (1 << type); | 167 | dso->loaded |= (1 << type); |
168 | } | 168 | } |
169 | 169 | ||
170 | void dso__sort_by_name(struct dso *self, enum map_type type); | 170 | void dso__sort_by_name(struct dso *dso, enum map_type type); |
171 | 171 | ||
172 | struct dso *__dsos__findnew(struct list_head *head, const char *name); | 172 | struct dso *__dsos__findnew(struct list_head *head, const char *name); |
173 | 173 | ||
174 | int dso__load(struct dso *self, struct map *map, symbol_filter_t filter); | 174 | int dso__load(struct dso *dso, struct map *map, symbol_filter_t filter); |
175 | int dso__load_vmlinux(struct dso *self, struct map *map, | 175 | int dso__load_vmlinux(struct dso *dso, struct map *map, |
176 | const char *vmlinux, symbol_filter_t filter); | 176 | const char *vmlinux, symbol_filter_t filter); |
177 | int dso__load_vmlinux_path(struct dso *self, struct map *map, | 177 | int dso__load_vmlinux_path(struct dso *dso, struct map *map, |
178 | symbol_filter_t filter); | 178 | symbol_filter_t filter); |
179 | int dso__load_kallsyms(struct dso *self, const char *filename, struct map *map, | 179 | int dso__load_kallsyms(struct dso *dso, const char *filename, struct map *map, |
180 | symbol_filter_t filter); | 180 | symbol_filter_t filter); |
181 | int machine__load_kallsyms(struct machine *self, const char *filename, | 181 | int machine__load_kallsyms(struct machine *machine, const char *filename, |
182 | enum map_type type, symbol_filter_t filter); | 182 | enum map_type type, symbol_filter_t filter); |
183 | int machine__load_vmlinux_path(struct machine *self, enum map_type type, | 183 | int machine__load_vmlinux_path(struct machine *machine, enum map_type type, |
184 | symbol_filter_t filter); | 184 | symbol_filter_t filter); |
185 | 185 | ||
186 | size_t __dsos__fprintf(struct list_head *head, FILE *fp); | 186 | size_t __dsos__fprintf(struct list_head *head, FILE *fp); |
187 | 187 | ||
188 | size_t machine__fprintf_dsos_buildid(struct machine *self, FILE *fp, bool with_hits); | 188 | size_t machine__fprintf_dsos_buildid(struct machine *machine, |
189 | size_t machines__fprintf_dsos(struct rb_root *self, FILE *fp); | 189 | FILE *fp, bool with_hits); |
190 | size_t machines__fprintf_dsos_buildid(struct rb_root *self, FILE *fp, bool with_hits); | 190 | size_t machines__fprintf_dsos(struct rb_root *machines, FILE *fp); |
191 | 191 | size_t machines__fprintf_dsos_buildid(struct rb_root *machines, | |
192 | size_t dso__fprintf_buildid(struct dso *self, FILE *fp); | 192 | FILE *fp, bool with_hits); |
193 | size_t dso__fprintf_symbols_by_name(struct dso *self, enum map_type type, FILE *fp); | 193 | size_t dso__fprintf_buildid(struct dso *dso, FILE *fp); |
194 | size_t dso__fprintf(struct dso *self, enum map_type type, FILE *fp); | 194 | size_t dso__fprintf_symbols_by_name(struct dso *dso, |
195 | enum map_type type, FILE *fp); | ||
196 | size_t dso__fprintf(struct dso *dso, enum map_type type, FILE *fp); | ||
195 | 197 | ||
196 | enum symtab_type { | 198 | enum symtab_type { |
197 | SYMTAB__KALLSYMS = 0, | 199 | SYMTAB__KALLSYMS = 0, |
@@ -207,34 +209,36 @@ enum symtab_type { | |||
207 | SYMTAB__NOT_FOUND, | 209 | SYMTAB__NOT_FOUND, |
208 | }; | 210 | }; |
209 | 211 | ||
210 | char dso__symtab_origin(const struct dso *self); | 212 | char dso__symtab_origin(const struct dso *dso); |
211 | void dso__set_long_name(struct dso *self, char *name); | 213 | void dso__set_long_name(struct dso *dso, char *name); |
212 | void dso__set_build_id(struct dso *self, void *build_id); | 214 | void dso__set_build_id(struct dso *dso, void *build_id); |
213 | void dso__read_running_kernel_build_id(struct dso *self, struct machine *machine); | 215 | void dso__read_running_kernel_build_id(struct dso *dso, |
214 | struct symbol *dso__find_symbol(struct dso *self, enum map_type type, u64 addr); | 216 | struct machine *machine); |
215 | struct symbol *dso__find_symbol_by_name(struct dso *self, enum map_type type, | 217 | struct symbol *dso__find_symbol(struct dso *dso, enum map_type type, |
218 | u64 addr); | ||
219 | struct symbol *dso__find_symbol_by_name(struct dso *dso, enum map_type type, | ||
216 | const char *name); | 220 | const char *name); |
217 | 221 | ||
218 | int filename__read_build_id(const char *filename, void *bf, size_t size); | 222 | int filename__read_build_id(const char *filename, void *bf, size_t size); |
219 | int sysfs__read_build_id(const char *filename, void *bf, size_t size); | 223 | int sysfs__read_build_id(const char *filename, void *bf, size_t size); |
220 | bool __dsos__read_build_ids(struct list_head *head, bool with_hits); | 224 | bool __dsos__read_build_ids(struct list_head *head, bool with_hits); |
221 | int build_id__sprintf(const u8 *self, int len, char *bf); | 225 | int build_id__sprintf(const u8 *build_id, int len, char *bf); |
222 | int kallsyms__parse(const char *filename, void *arg, | 226 | int kallsyms__parse(const char *filename, void *arg, |
223 | int (*process_symbol)(void *arg, const char *name, | 227 | int (*process_symbol)(void *arg, const char *name, |
224 | char type, u64 start, u64 end)); | 228 | char type, u64 start, u64 end)); |
225 | 229 | ||
226 | void machine__destroy_kernel_maps(struct machine *self); | 230 | void machine__destroy_kernel_maps(struct machine *machine); |
227 | int __machine__create_kernel_maps(struct machine *self, struct dso *kernel); | 231 | int __machine__create_kernel_maps(struct machine *machine, struct dso *kernel); |
228 | int machine__create_kernel_maps(struct machine *self); | 232 | int machine__create_kernel_maps(struct machine *machine); |
229 | 233 | ||
230 | int machines__create_kernel_maps(struct rb_root *self, pid_t pid); | 234 | int machines__create_kernel_maps(struct rb_root *machines, pid_t pid); |
231 | int machines__create_guest_kernel_maps(struct rb_root *self); | 235 | int machines__create_guest_kernel_maps(struct rb_root *machines); |
232 | void machines__destroy_guest_kernel_maps(struct rb_root *self); | 236 | void machines__destroy_guest_kernel_maps(struct rb_root *machines); |
233 | 237 | ||
234 | int symbol__init(void); | 238 | int symbol__init(void); |
235 | void symbol__exit(void); | 239 | void symbol__exit(void); |
236 | bool symbol_type__is_a(char symbol_type, enum map_type map_type); | 240 | bool symbol_type__is_a(char symbol_type, enum map_type map_type); |
237 | 241 | ||
238 | size_t machine__fprintf_vmlinux_path(struct machine *self, FILE *fp); | 242 | size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp); |
239 | 243 | ||
240 | #endif /* __PERF_SYMBOL */ | 244 | #endif /* __PERF_SYMBOL */ |