aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2011-05-19 20:36:08 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2011-05-19 20:36:08 -0400
commitdf48d8716eab9608fe93924e4ae06ff110e8674f (patch)
tree0fe10733a414b3651e1dae29518b7960a4da0aa4
parentacd30250d7d0f495685d1c7c6184636a22fcdf7f (diff)
parent29510ec3b626c86de9707bb8904ff940d430289b (diff)
Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (107 commits) perf stat: Add more cache-miss percentage printouts perf stat: Add -d -d and -d -d -d options to show more CPU events ftrace/kbuild: Add recordmcount files to force full build ftrace: Add self-tests for multiple function trace users ftrace: Modify ftrace_set_filter/notrace to take ops ftrace: Allow dynamically allocated function tracers ftrace: Implement separate user function filtering ftrace: Free hash with call_rcu_sched() ftrace: Have global_ops store the functions that are to be traced ftrace: Add ops parameter to ftrace_startup/shutdown functions ftrace: Add enabled_functions file ftrace: Use counters to enable functions to trace ftrace: Separate hash allocation and assignment ftrace: Create a global_ops to hold the filter and notrace hashes ftrace: Use hash instead for FTRACE_FL_FILTER ftrace: Replace FTRACE_FL_NOTRACE flag with a hash of ignored functions perf bench, x86: Add alternatives-asm.h wrapper x86, 64-bit: Fix copy_[to/from]_user() checks for the userspace address limit x86, mem: memset_64.S: Optimize memset by enhanced REP MOVSB/STOSB x86, mem: memmove_64.S: Optimize memmove by enhanced REP MOVSB/STOSB ...
Diffstat
-rw-r--r--Makefile1
-rw-r--r--arch/mips/include/asm/jump_label.h22
-rw-r--r--arch/s390/Kconfig1
-rw-r--r--arch/s390/include/asm/ftrace.h4
-rw-r--r--arch/s390/include/asm/jump_label.h37
-rw-r--r--arch/s390/kernel/Makefile2
-rw-r--r--arch/s390/kernel/jump_label.c59
-rw-r--r--arch/sparc/include/asm/jump_label.h25
-rw-r--r--arch/x86/include/asm/alternative-asm.h9
-rw-r--r--arch/x86/include/asm/alternative.h3
-rw-r--r--arch/x86/include/asm/cpufeature.h1
-rw-r--r--arch/x86/include/asm/ftrace.h7
-rw-r--r--arch/x86/include/asm/jump_label.h27
-rw-r--r--arch/x86/include/asm/setup.h2
-rw-r--r--arch/x86/include/asm/stacktrace.h3
-rw-r--r--arch/x86/include/asm/uaccess.h2
-rw-r--r--arch/x86/kernel/alternative.c11
-rw-r--r--arch/x86/kernel/cpu/common.c3
-rw-r--r--arch/x86/kernel/cpu/intel.c19
-rw-r--r--arch/x86/kernel/cpu/perf_event.c28
-rw-r--r--arch/x86/kernel/cpu/perf_event_amd.c14
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel.c37
-rw-r--r--arch/x86/kernel/cpu/perf_event_p4.c7
-rw-r--r--arch/x86/kernel/dumpstack.c16
-rw-r--r--arch/x86/kernel/module.c1
-rw-r--r--arch/x86/kernel/stacktrace.c13
-rw-r--r--arch/x86/lib/clear_page_64.S33
-rw-r--r--arch/x86/lib/copy_user_64.S69
-rw-r--r--arch/x86/lib/memcpy_64.S45
-rw-r--r--arch/x86/lib/memmove_64.S29
-rw-r--r--arch/x86/lib/memset_64.S54
-rw-r--r--arch/x86/oprofile/backtrace.c13
-rw-r--r--include/asm-generic/vmlinux.lds.h14
-rw-r--r--include/linux/dynamic_debug.h2
-rw-r--r--include/linux/ftrace.h35
-rw-r--r--include/linux/init.h14
-rw-r--r--include/linux/jump_label.h89
-rw-r--r--include/linux/jump_label_ref.h44
-rw-r--r--include/linux/kernel.h1
-rw-r--r--include/linux/perf_event.h121
-rw-r--r--include/linux/tracepoint.h22
-rw-r--r--kernel/Makefile6
-rw-r--r--kernel/events/Makefile6
-rw-r--r--kernel/events/core.c (renamed from kernel/perf_event.c)44
-rw-r--r--kernel/events/hw_breakpoint.c (renamed from kernel/hw_breakpoint.c)0
-rw-r--r--kernel/extable.c8
-rw-r--r--kernel/jump_label.c539
-rw-r--r--kernel/trace/ftrace.c1261
-rw-r--r--kernel/trace/trace.c15
-rw-r--r--kernel/trace/trace.h2
-rw-r--r--kernel/trace/trace_functions.c2
-rw-r--r--kernel/trace/trace_irqsoff.c1
-rw-r--r--kernel/trace/trace_output.c3
-rw-r--r--kernel/trace/trace_printk.c120
-rw-r--r--kernel/trace/trace_sched_wakeup.c1
-rw-r--r--kernel/trace/trace_selftest.c214
-rw-r--r--kernel/trace/trace_selftest_dynamic.c6
-rw-r--r--kernel/trace/trace_stack.c1
-rw-r--r--kernel/tracepoint.c23
-rw-r--r--scripts/Makefile.build12
-rw-r--r--scripts/recordmcount.c168
-rw-r--r--scripts/recordmcount.h174
-rwxr-xr-xscripts/recordmcount.pl5
-rw-r--r--tools/perf/Documentation/perf-script.txt52
-rw-r--r--tools/perf/Makefile107
-rw-r--r--tools/perf/builtin-script.c296
-rw-r--r--tools/perf/builtin-stat.c573
-rw-r--r--tools/perf/config/feature-tests.mak (renamed from tools/perf/feature-tests.mak)16
-rw-r--r--tools/perf/config/utilities.mak188
-rw-r--r--tools/perf/util/include/asm/alternative-asm.h8
-rw-r--r--tools/perf/util/parse-events.c123
-rw-r--r--tools/perf/util/probe-finder.c73
-rw-r--r--tools/perf/util/probe-finder.h2
-rw-r--r--tools/perf/util/python.c3
-rw-r--r--tools/perf/util/session.c12
-rw-r--r--tools/perf/util/session.h3
-rw-r--r--tools/perf/util/symbol.c629
-rw-r--r--tools/perf/util/symbol.h78
78 files changed, 4013 insertions, 1700 deletions
diff --git a/Makefile b/Makefile
index 123d858dae03..a0344a81a893 100644
--- a/Makefile
+++ b/Makefile
@@ -1268,6 +1268,7 @@ help:
1268 @echo ' make C=1 [targets] Check all c source with $$CHECK (sparse by default)' 1268 @echo ' make C=1 [targets] Check all c source with $$CHECK (sparse by default)'
1269 @echo ' make C=2 [targets] Force check of all c source with $$CHECK' 1269 @echo ' make C=2 [targets] Force check of all c source with $$CHECK'
1270 @echo ' make W=1 [targets] Enable extra gcc checks' 1270 @echo ' make W=1 [targets] Enable extra gcc checks'
1271 @echo ' make RECORDMCOUNT_WARN=1 [targets] Warn about ignored mcount sections'
1271 @echo '' 1272 @echo ''
1272 @echo 'Execute "make" or "make all" to build all targets marked with [*] ' 1273 @echo 'Execute "make" or "make all" to build all targets marked with [*] '
1273 @echo 'For further info see the ./README file' 1274 @echo 'For further info see the ./README file'
diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h
index 7622ccf75076..1881b316ca45 100644
--- a/arch/mips/include/asm/jump_label.h
+++ b/arch/mips/include/asm/jump_label.h
@@ -20,16 +20,18 @@
20#define WORD_INSN ".word" 20#define WORD_INSN ".word"
21#endif 21#endif
22 22
23#define JUMP_LABEL(key, label) \ 23static __always_inline bool arch_static_branch(struct jump_label_key *key)
24 do { \ 24{
25 asm goto("1:\tnop\n\t" \ 25 asm goto("1:\tnop\n\t"
26 "nop\n\t" \ 26 "nop\n\t"
27 ".pushsection __jump_table, \"a\"\n\t" \ 27 ".pushsection __jump_table, \"aw\"\n\t"
28 WORD_INSN " 1b, %l[" #label "], %0\n\t" \ 28 WORD_INSN " 1b, %l[l_yes], %0\n\t"
29 ".popsection\n\t" \ 29 ".popsection\n\t"
30 : : "i" (key) : : label); \ 30 : : "i" (key) : : l_yes);
31 } while (0) 31 return false;
32 32l_yes:
33 return true;
34}
33 35
34#endif /* __KERNEL__ */ 36#endif /* __KERNEL__ */
35 37
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 2508a6f31588..4a7f14079e03 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -88,6 +88,7 @@ config S390
88 select HAVE_KERNEL_XZ 88 select HAVE_KERNEL_XZ
89 select HAVE_GET_USER_PAGES_FAST 89 select HAVE_GET_USER_PAGES_FAST
90 select HAVE_ARCH_MUTEX_CPU_RELAX 90 select HAVE_ARCH_MUTEX_CPU_RELAX
91 select HAVE_ARCH_JUMP_LABEL if !MARCH_G5
91 select ARCH_INLINE_SPIN_TRYLOCK 92 select ARCH_INLINE_SPIN_TRYLOCK
92 select ARCH_INLINE_SPIN_TRYLOCK_BH 93 select ARCH_INLINE_SPIN_TRYLOCK_BH
93 select ARCH_INLINE_SPIN_LOCK 94 select ARCH_INLINE_SPIN_LOCK
diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h
index 3c29be4836ed..b7931faaef6d 100644
--- a/arch/s390/include/asm/ftrace.h
+++ b/arch/s390/include/asm/ftrace.h
@@ -11,15 +11,13 @@ struct dyn_arch_ftrace { };
11 11
12#ifdef CONFIG_64BIT 12#ifdef CONFIG_64BIT
13#define MCOUNT_INSN_SIZE 12 13#define MCOUNT_INSN_SIZE 12
14#define MCOUNT_OFFSET 8
15#else 14#else
16#define MCOUNT_INSN_SIZE 20 15#define MCOUNT_INSN_SIZE 20
17#define MCOUNT_OFFSET 4
18#endif 16#endif
19 17
20static inline unsigned long ftrace_call_adjust(unsigned long addr) 18static inline unsigned long ftrace_call_adjust(unsigned long addr)
21{ 19{
22 return addr - MCOUNT_OFFSET; 20 return addr;
23} 21}
24 22
25#endif /* __ASSEMBLY__ */ 23#endif /* __ASSEMBLY__ */
diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h
new file mode 100644
index 000000000000..95a6cf2b5b67
--- /dev/null
+++ b/arch/s390/include/asm/jump_label.h
@@ -0,0 +1,37 @@
1#ifndef _ASM_S390_JUMP_LABEL_H
2#define _ASM_S390_JUMP_LABEL_H
3
4#include <linux/types.h>
5
6#define JUMP_LABEL_NOP_SIZE 6
7
8#ifdef CONFIG_64BIT
9#define ASM_PTR ".quad"
10#define ASM_ALIGN ".balign 8"
11#else
12#define ASM_PTR ".long"
13#define ASM_ALIGN ".balign 4"
14#endif
15
16static __always_inline bool arch_static_branch(struct jump_label_key *key)
17{
18 asm goto("0: brcl 0,0\n"
19 ".pushsection __jump_table, \"aw\"\n"
20 ASM_ALIGN "\n"
21 ASM_PTR " 0b, %l[label], %0\n"
22 ".popsection\n"
23 : : "X" (key) : : label);
24 return false;
25label:
26 return true;
27}
28
29typedef unsigned long jump_label_t;
30
31struct jump_entry {
32 jump_label_t code;
33 jump_label_t target;
34 jump_label_t key;
35};
36
37#endif
diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile
index 64230bc392fa..5ff15dacb571 100644
--- a/arch/s390/kernel/Makefile
+++ b/arch/s390/kernel/Makefile
@@ -23,7 +23,7 @@ CFLAGS_sysinfo.o += -Iinclude/math-emu -Iarch/s390/math-emu -w
23obj-y := bitmap.o traps.o time.o process.o base.o early.o setup.o \ 23obj-y := bitmap.o traps.o time.o process.o base.o early.o setup.o \
24 processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o \ 24 processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o \
25 s390_ext.o debug.o irq.o ipl.o dis.o diag.o mem_detect.o \ 25 s390_ext.o debug.o irq.o ipl.o dis.o diag.o mem_detect.o \
26 vdso.o vtime.o sysinfo.o nmi.o sclp.o 26 vdso.o vtime.o sysinfo.o nmi.o sclp.o jump_label.o
27 27
28obj-y += $(if $(CONFIG_64BIT),entry64.o,entry.o) 28obj-y += $(if $(CONFIG_64BIT),entry64.o,entry.o)
29obj-y += $(if $(CONFIG_64BIT),reipl64.o,reipl.o) 29obj-y += $(if $(CONFIG_64BIT),reipl64.o,reipl.o)
diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c
new file mode 100644
index 000000000000..44cc06bedf77
--- /dev/null
+++ b/arch/s390/kernel/jump_label.c
@@ -0,0 +1,59 @@
1/*
2 * Jump label s390 support
3 *
4 * Copyright IBM Corp. 2011
5 * Author(s): Jan Glauber <jang@linux.vnet.ibm.com>
6 */
7#include <linux/module.h>
8#include <linux/uaccess.h>
9#include <linux/stop_machine.h>
10#include <linux/jump_label.h>
11#include <asm/ipl.h>
12
13#ifdef HAVE_JUMP_LABEL
14
15struct insn {
16 u16 opcode;
17 s32 offset;
18} __packed;
19
20struct insn_args {
21 unsigned long *target;
22 struct insn *insn;
23 ssize_t size;
24};
25
26static int __arch_jump_label_transform(void *data)
27{
28 struct insn_args *args = data;
29 int rc;
30
31 rc = probe_kernel_write(args->target, args->insn, args->size);
32 WARN_ON_ONCE(rc < 0);
33 return 0;
34}
35
36void arch_jump_label_transform(struct jump_entry *entry,
37 enum jump_label_type type)
38{
39 struct insn_args args;
40 struct insn insn;
41
42 if (type == JUMP_LABEL_ENABLE) {
43 /* brcl 15,offset */
44 insn.opcode = 0xc0f4;
45 insn.offset = (entry->target - entry->code) >> 1;
46 } else {
47 /* brcl 0,0 */
48 insn.opcode = 0xc004;
49 insn.offset = 0;
50 }
51
52 args.target = (void *) entry->code;
53 args.insn = &insn;
54 args.size = JUMP_LABEL_NOP_SIZE;
55
56 stop_machine(__arch_jump_label_transform, &args, NULL);
57}
58
59#endif
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h
index 427d4684e0d2..fc73a82366f8 100644
--- a/arch/sparc/include/asm/jump_label.h
+++ b/arch/sparc/include/asm/jump_label.h
@@ -7,17 +7,20 @@
7 7
8#define JUMP_LABEL_NOP_SIZE 4 8#define JUMP_LABEL_NOP_SIZE 4
9 9
10#define JUMP_LABEL(key, label) \ 10static __always_inline bool arch_static_branch(struct jump_label_key *key)
11 do { \ 11{
12 asm goto("1:\n\t" \ 12 asm goto("1:\n\t"
13 "nop\n\t" \ 13 "nop\n\t"
14 "nop\n\t" \ 14 "nop\n\t"
15 ".pushsection __jump_table, \"a\"\n\t"\ 15 ".pushsection __jump_table, \"aw\"\n\t"
16 ".align 4\n\t" \ 16 ".align 4\n\t"
17 ".word 1b, %l[" #label "], %c0\n\t" \ 17 ".word 1b, %l[l_yes], %c0\n\t"
18 ".popsection \n\t" \ 18 ".popsection \n\t"
19 : : "i" (key) : : label);\ 19 : : "i" (key) : : l_yes);
20 } while (0) 20 return false;
21l_yes:
22 return true;
23}
21 24
22#endif /* __KERNEL__ */ 25#endif /* __KERNEL__ */
23 26
diff --git a/arch/x86/include/asm/alternative-asm.h b/arch/x86/include/asm/alternative-asm.h
index a63a68be1cce..94d420b360d1 100644
--- a/arch/x86/include/asm/alternative-asm.h
+++ b/arch/x86/include/asm/alternative-asm.h
@@ -15,4 +15,13 @@
15 .endm 15 .endm
16#endif 16#endif
17 17
18.macro altinstruction_entry orig alt feature orig_len alt_len
19 .align 8
20 .quad \orig
21 .quad \alt
22 .word \feature
23 .byte \orig_len
24 .byte \alt_len
25.endm
26
18#endif /* __ASSEMBLY__ */ 27#endif /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 13009d1af99a..8cdd1e247975 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -4,7 +4,6 @@
4#include <linux/types.h> 4#include <linux/types.h>
5#include <linux/stddef.h> 5#include <linux/stddef.h>
6#include <linux/stringify.h> 6#include <linux/stringify.h>
7#include <linux/jump_label.h>
8#include <asm/asm.h> 7#include <asm/asm.h>
9 8
10/* 9/*
@@ -191,7 +190,7 @@ extern void *text_poke(void *addr, const void *opcode, size_t len);
191extern void *text_poke_smp(void *addr, const void *opcode, size_t len); 190extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
192extern void text_poke_smp_batch(struct text_poke_param *params, int n); 191extern void text_poke_smp_batch(struct text_poke_param *params, int n);
193 192
194#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) 193#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL)
195#define IDEAL_NOP_SIZE_5 5 194#define IDEAL_NOP_SIZE_5 5
196extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5]; 195extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
197extern void arch_init_ideal_nop5(void); 196extern void arch_init_ideal_nop5(void);
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 91f3e087cf21..7f2f7b123293 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -195,6 +195,7 @@
195 195
196/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */ 196/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */
197#define X86_FEATURE_FSGSBASE (9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/ 197#define X86_FEATURE_FSGSBASE (9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/
198#define X86_FEATURE_ERMS (9*32+ 9) /* Enhanced REP MOVSB/STOSB */
198 199
199#if defined(__KERNEL__) && !defined(__ASSEMBLY__) 200#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
200 201
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index db24c2278be0..268c783ab1c0 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -38,11 +38,10 @@ extern void mcount(void);
38static inline unsigned long ftrace_call_adjust(unsigned long addr) 38static inline unsigned long ftrace_call_adjust(unsigned long addr)
39{ 39{
40 /* 40 /*
41 * call mcount is "e8 <4 byte offset>" 41 * addr is the address of the mcount call instruction.
42 * The addr points to the 4 byte offset and the caller of this 42 * recordmcount does the necessary offset calculation.
43 * function wants the pointer to e8. Simply subtract one.
44 */ 43 */
45 return addr - 1; 44 return addr;
46} 45}
47 46
48#ifdef CONFIG_DYNAMIC_FTRACE 47#ifdef CONFIG_DYNAMIC_FTRACE
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index 574dbc22893a..a32b18ce6ead 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -5,20 +5,25 @@
5 5
6#include <linux/types.h> 6#include <linux/types.h>
7#include <asm/nops.h> 7#include <asm/nops.h>
8#include <asm/asm.h>
8 9
9#define JUMP_LABEL_NOP_SIZE 5 10#define JUMP_LABEL_NOP_SIZE 5
10 11
11# define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" 12#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
12 13
13# define JUMP_LABEL(key, label) \ 14static __always_inline bool arch_static_branch(struct jump_label_key *key)
14 do { \ 15{
15 asm goto("1:" \ 16 asm goto("1:"
16 JUMP_LABEL_INITIAL_NOP \ 17 JUMP_LABEL_INITIAL_NOP
17 ".pushsection __jump_table, \"aw\" \n\t"\ 18 ".pushsection __jump_table, \"aw\" \n\t"
18 _ASM_PTR "1b, %l[" #label "], %c0 \n\t" \ 19 _ASM_ALIGN "\n\t"
19 ".popsection \n\t" \ 20 _ASM_PTR "1b, %l[l_yes], %c0 \n\t"
20 : : "i" (key) : : label); \ 21 ".popsection \n\t"
21 } while (0) 22 : : "i" (key) : : l_yes);
23 return false;
24l_yes:
25 return true;
26}
22 27
23#endif /* __KERNEL__ */ 28#endif /* __KERNEL__ */
24 29
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index db8aa19a08a2..647d8a06ce4f 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -88,7 +88,7 @@ void *extend_brk(size_t size, size_t align);
88 * executable.) 88 * executable.)
89 */ 89 */
90#define RESERVE_BRK(name,sz) \ 90#define RESERVE_BRK(name,sz) \
91 static void __section(.discard.text) __used \ 91 static void __section(.discard.text) __used notrace \
92 __brk_reservation_fn_##name##__(void) { \ 92 __brk_reservation_fn_##name##__(void) { \
93 asm volatile ( \ 93 asm volatile ( \
94 ".pushsection .brk_reservation,\"aw\",@nobits;" \ 94 ".pushsection .brk_reservation,\"aw\",@nobits;" \
diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h
index d7e89c83645d..70bbe39043a9 100644
--- a/arch/x86/include/asm/stacktrace.h
+++ b/arch/x86/include/asm/stacktrace.h
@@ -37,9 +37,6 @@ print_context_stack_bp(struct thread_info *tinfo,
37/* Generic stack tracer with callbacks */ 37/* Generic stack tracer with callbacks */
38 38
39struct stacktrace_ops { 39struct stacktrace_ops {
40 void (*warning)(void *data, char *msg);
41 /* msg must contain %s for the symbol */
42 void (*warning_symbol)(void *data, char *msg, unsigned long symbol);
43 void (*address)(void *data, unsigned long address, int reliable); 40 void (*address)(void *data, unsigned long address, int reliable);
44 /* On negative return stop dumping */ 41 /* On negative return stop dumping */
45 int (*stack)(void *data, char *name); 42 int (*stack)(void *data, char *name);
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index abd3e0ea762a..99f0ad753f32 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -42,7 +42,7 @@
42 * Returns 0 if the range is valid, nonzero otherwise. 42 * Returns 0 if the range is valid, nonzero otherwise.
43 * 43 *
44 * This is equivalent to the following test: 44 * This is equivalent to the following test:
45 * (u33)addr + (u33)size >= (u33)current->addr_limit.seg (u65 for x86_64) 45 * (u33)addr + (u33)size > (u33)current->addr_limit.seg (u65 for x86_64)
46 * 46 *
47 * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry... 47 * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry...
48 */ 48 */
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 4a234677e213..1eeeafcb4410 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -210,6 +210,15 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
210 u8 insnbuf[MAX_PATCH_LEN]; 210 u8 insnbuf[MAX_PATCH_LEN];
211 211
212 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); 212 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);
213 /*
214 * The scan order should be from start to end. A later scanned
215 * alternative code can overwrite a previous scanned alternative code.
216 * Some kernel functions (e.g. memcpy, memset, etc) use this order to
217 * patch code.
218 *
219 * So be careful if you want to change the scan order to any other
220 * order.
221 */
213 for (a = start; a < end; a++) { 222 for (a = start; a < end; a++) {
214 u8 *instr = a->instr; 223 u8 *instr = a->instr;
215 BUG_ON(a->replacementlen > a->instrlen); 224 BUG_ON(a->replacementlen > a->instrlen);
@@ -679,7 +688,7 @@ void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n)
679 __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); 688 __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL);
680} 689}
681 690
682#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) 691#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL)
683 692
684#ifdef CONFIG_X86_64 693#ifdef CONFIG_X86_64
685unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 }; 694unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 };
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index e2ced0074a45..173f3a3fa1a6 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -565,8 +565,7 @@ void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c)
565 565
566 cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx); 566 cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
567 567
568 if (eax > 0) 568 c->x86_capability[9] = ebx;
569 c->x86_capability[9] = ebx;
570 } 569 }
571 570
572 /* AMD-defined flags: level 0x80000001 */ 571 /* AMD-defined flags: level 0x80000001 */
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index df86bc8c859d..fc73a34ba8c9 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -29,10 +29,10 @@
29 29
30static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) 30static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
31{ 31{
32 u64 misc_enable;
33
32 /* Unmask CPUID levels if masked: */ 34 /* Unmask CPUID levels if masked: */
33 if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) { 35 if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) {
34 u64 misc_enable;
35
36 rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable); 36 rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
37 37
38 if (misc_enable & MSR_IA32_MISC_ENABLE_LIMIT_CPUID) { 38 if (misc_enable & MSR_IA32_MISC_ENABLE_LIMIT_CPUID) {
@@ -118,8 +118,6 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
118 * (model 2) with the same problem. 118 * (model 2) with the same problem.
119 */ 119 */
120 if (c->x86 == 15) { 120 if (c->x86 == 15) {
121 u64 misc_enable;
122
123 rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable); 121 rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
124 122
125 if (misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING) { 123 if (misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING) {
@@ -130,6 +128,19 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
130 } 128 }
131 } 129 }
132#endif 130#endif
131
132 /*
133 * If fast string is not enabled in IA32_MISC_ENABLE for any reason,
134 * clear the fast string and enhanced fast string CPU capabilities.
135 */
136 if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) {
137 rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
138 if (!(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING)) {
139 printk(KERN_INFO "Disabled fast string operations\n");
140 setup_clear_cpu_cap(X86_FEATURE_REP_GOOD);
141 setup_clear_cpu_cap(X86_FEATURE_ERMS);
142 }
143 }
133} 144}
134 145
135#ifdef CONFIG_X86_32 146#ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index e638689279d3..3a0338b4b179 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -31,6 +31,7 @@
31#include <asm/nmi.h> 31#include <asm/nmi.h>
32#include <asm/compat.h> 32#include <asm/compat.h>
33#include <asm/smp.h> 33#include <asm/smp.h>
34#include <asm/alternative.h>
34 35
35#if 0 36#if 0
36#undef wrmsrl 37#undef wrmsrl
@@ -363,12 +364,18 @@ again:
363 return new_raw_count; 364 return new_raw_count;
364} 365}
365 366
366/* using X86_FEATURE_PERFCTR_CORE to later implement ALTERNATIVE() here */
367static inline int x86_pmu_addr_offset(int index) 367static inline int x86_pmu_addr_offset(int index)
368{ 368{
369 if (boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) 369 int offset;
370 return index << 1; 370
371 return index; 371 /* offset = X86_FEATURE_PERFCTR_CORE ? index << 1 : index */
372 alternative_io(ASM_NOP2,
373 "shll $1, %%eax",
374 X86_FEATURE_PERFCTR_CORE,
375 "=a" (offset),
376 "a" (index));
377
378 return offset;
372} 379}
373 380
374static inline unsigned int x86_pmu_config_addr(int index) 381static inline unsigned int x86_pmu_config_addr(int index)
@@ -1766,17 +1773,6 @@ static struct pmu pmu = {
1766 * callchain support 1773 * callchain support
1767 */ 1774 */
1768 1775
1769static void
1770backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1771{
1772 /* Ignore warnings */
1773}
1774
1775static void backtrace_warning(void *data, char *msg)
1776{
1777 /* Ignore warnings */
1778}
1779
1780static int backtrace_stack(void *data, char *name) 1776static int backtrace_stack(void *data, char *name)
1781{ 1777{
1782 return 0; 1778 return 0;
@@ -1790,8 +1786,6 @@ static void backtrace_address(void *data, unsigned long addr, int reliable)
1790} 1786}
1791 1787
1792static const struct stacktrace_ops backtrace_ops = { 1788static const struct stacktrace_ops backtrace_ops = {
1793 .warning = backtrace_warning,
1794 .warning_symbol = backtrace_warning_symbol,
1795 .stack = backtrace_stack, 1789 .stack = backtrace_stack,
1796 .address = backtrace_address, 1790 .address = backtrace_address,
1797 .walk_stack = print_context_stack_bp, 1791 .walk_stack = print_context_stack_bp,
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index cf4e369cea67..fe29c1d2219e 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -96,12 +96,14 @@ static __initconst const u64 amd_hw_cache_event_ids
96 */ 96 */
97static const u64 amd_perfmon_event_map[] = 97static const u64 amd_perfmon_event_map[] =
98{ 98{
99 [PERF_COUNT_HW_CPU_CYCLES] = 0x0076, 99 [PERF_COUNT_HW_CPU_CYCLES] = 0x0076,
100 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, 100 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
101 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080, 101 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080,
102 [PERF_COUNT_HW_CACHE_MISSES] = 0x0081, 102 [PERF_COUNT_HW_CACHE_MISSES] = 0x0081,
103 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2, 103 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2,
104 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3, 104 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3,
105 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */
106 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x00d1, /* "Dispatch stalls" event */
105}; 107};
106 108
107static u64 amd_pmu_event_map(int hw_event) 109static u64 amd_pmu_event_map(int hw_event)
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 447a28de6f09..41178c826c48 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -36,7 +36,7 @@ static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly =
36 [PERF_COUNT_HW_BUS_CYCLES] = 0x013c, 36 [PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
37}; 37};
38 38
39static struct event_constraint intel_core_event_constraints[] = 39static struct event_constraint intel_core_event_constraints[] __read_mostly =
40{ 40{
41 INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */ 41 INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
42 INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */ 42 INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
@@ -47,7 +47,7 @@ static struct event_constraint intel_core_event_constraints[] =
47 EVENT_CONSTRAINT_END 47 EVENT_CONSTRAINT_END
48}; 48};
49 49
50static struct event_constraint intel_core2_event_constraints[] = 50static struct event_constraint intel_core2_event_constraints[] __read_mostly =
51{ 51{
52 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ 52 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
53 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ 53 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@@ -70,7 +70,7 @@ static struct event_constraint intel_core2_event_constraints[] =
70 EVENT_CONSTRAINT_END 70 EVENT_CONSTRAINT_END
71}; 71};
72 72
73static struct event_constraint intel_nehalem_event_constraints[] = 73static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =
74{ 74{
75 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ 75 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
76 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ 76 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@@ -86,19 +86,19 @@ static struct event_constraint intel_nehalem_event_constraints[] =
86 EVENT_CONSTRAINT_END 86 EVENT_CONSTRAINT_END
87}; 87};
88 88
89static struct extra_reg intel_nehalem_extra_regs[] = 89static struct extra_reg intel_nehalem_extra_regs[] __read_mostly =
90{ 90{
91 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff), 91 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
92 EVENT_EXTRA_END 92 EVENT_EXTRA_END
93}; 93};
94 94
95static struct event_constraint intel_nehalem_percore_constraints[] = 95static struct event_constraint intel_nehalem_percore_constraints[] __read_mostly =
96{ 96{
97 INTEL_EVENT_CONSTRAINT(0xb7, 0), 97 INTEL_EVENT_CONSTRAINT(0xb7, 0),
98 EVENT_CONSTRAINT_END 98 EVENT_CONSTRAINT_END
99}; 99};
100 100
101static struct event_constraint intel_westmere_event_constraints[] = 101static struct event_constraint intel_westmere_event_constraints[] __read_mostly =
102{ 102{
103 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ 103 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
104 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ 104 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@@ -110,7 +110,7 @@ static struct event_constraint intel_westmere_event_constraints[] =
110 EVENT_CONSTRAINT_END 110 EVENT_CONSTRAINT_END
111}; 111};
112 112
113static struct event_constraint intel_snb_event_constraints[] = 113static struct event_constraint intel_snb_event_constraints[] __read_mostly =
114{ 114{
115 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ 115 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
116 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ 116 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@@ -123,21 +123,21 @@ static struct event_constraint intel_snb_event_constraints[] =
123 EVENT_CONSTRAINT_END 123 EVENT_CONSTRAINT_END
124}; 124};
125 125
126static struct extra_reg intel_westmere_extra_regs[] = 126static struct extra_reg intel_westmere_extra_regs[] __read_mostly =
127{ 127{
128 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff), 128 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
129 INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff), 129 INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff),
130 EVENT_EXTRA_END 130 EVENT_EXTRA_END
131}; 131};
132 132
133static struct event_constraint intel_westmere_percore_constraints[] = 133static struct event_constraint intel_westmere_percore_constraints[] __read_mostly =
134{ 134{
135 INTEL_EVENT_CONSTRAINT(0xb7, 0), 135 INTEL_EVENT_CONSTRAINT(0xb7, 0),
136 INTEL_EVENT_CONSTRAINT(0xbb, 0), 136 INTEL_EVENT_CONSTRAINT(0xbb, 0),
137 EVENT_CONSTRAINT_END 137 EVENT_CONSTRAINT_END
138}; 138};
139 139
140static struct event_constraint intel_gen_event_constraints[] = 140static struct event_constraint intel_gen_event_constraints[] __read_mostly =
141{ 141{
142 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ 142 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
143 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ 143 FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@@ -1440,6 +1440,11 @@ static __init int intel_pmu_init(void)
1440 x86_pmu.enable_all = intel_pmu_nhm_enable_all; 1440 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
1441 x86_pmu.extra_regs = intel_nehalem_extra_regs; 1441 x86_pmu.extra_regs = intel_nehalem_extra_regs;
1442 1442
1443 /* UOPS_ISSUED.STALLED_CYCLES */
1444 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e;
1445 /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
1446 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1;
1447
1443 if (ebx & 0x40) { 1448 if (ebx & 0x40) {
1444 /* 1449 /*
1445 * Erratum AAJ80 detected, we work it around by using 1450 * Erratum AAJ80 detected, we work it around by using
@@ -1480,6 +1485,12 @@ static __init int intel_pmu_init(void)
1480 x86_pmu.enable_all = intel_pmu_nhm_enable_all; 1485 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
1481 x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints; 1486 x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints;
1482 x86_pmu.extra_regs = intel_westmere_extra_regs; 1487 x86_pmu.extra_regs = intel_westmere_extra_regs;
1488
1489 /* UOPS_ISSUED.STALLED_CYCLES */
1490 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e;
1491 /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
1492 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1;
1493
1483 pr_cont("Westmere events, "); 1494 pr_cont("Westmere events, ");
1484 break; 1495 break;
1485 1496
@@ -1491,6 +1502,12 @@ static __init int intel_pmu_init(void)
1491 1502
1492 x86_pmu.event_constraints = intel_snb_event_constraints; 1503 x86_pmu.event_constraints = intel_snb_event_constraints;
1493 x86_pmu.pebs_constraints = intel_snb_pebs_events; 1504 x86_pmu.pebs_constraints = intel_snb_pebs_events;
1505
1506 /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
1507 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e;
1508 /* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/
1509 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x18001b1;
1510
1494 pr_cont("SandyBridge events, "); 1511 pr_cont("SandyBridge events, ");
1495 break; 1512 break;
1496 1513
diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c
index e93fcd55fae1..ead584fb6a7d 100644
--- a/arch/x86/kernel/cpu/perf_event_p4.c
+++ b/arch/x86/kernel/cpu/perf_event_p4.c
@@ -468,7 +468,7 @@ static struct p4_event_bind p4_event_bind_map[] = {
468 .opcode = P4_OPCODE(P4_EVENT_MISPRED_BRANCH_RETIRED), 468 .opcode = P4_OPCODE(P4_EVENT_MISPRED_BRANCH_RETIRED),
469 .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 }, 469 .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
470 .escr_emask = 470 .escr_emask =
471 P4_ESCR_EMASK_BIT(P4_EVENT_MISPRED_BRANCH_RETIRED, NBOGUS), 471 P4_ESCR_EMASK_BIT(P4_EVENT_MISPRED_BRANCH_RETIRED, NBOGUS),
472 .cntr = { {12, 13, 16}, {14, 15, 17} }, 472 .cntr = { {12, 13, 16}, {14, 15, 17} },
473 }, 473 },
474 [P4_EVENT_X87_ASSIST] = { 474 [P4_EVENT_X87_ASSIST] = {
@@ -912,8 +912,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs)
912 int idx, handled = 0; 912 int idx, handled = 0;
913 u64 val; 913 u64 val;
914 914
915 data.addr = 0; 915 perf_sample_data_init(&data, 0);
916 data.raw = NULL;
917 916
918 cpuc = &__get_cpu_var(cpu_hw_events); 917 cpuc = &__get_cpu_var(cpu_hw_events);
919 918
@@ -1197,7 +1196,7 @@ static __init int p4_pmu_init(void)
1197{ 1196{
1198 unsigned int low, high; 1197 unsigned int low, high;
1199 1198
1200 /* If we get stripped -- indexig fails */ 1199 /* If we get stripped -- indexing fails */
1201 BUILD_BUG_ON(ARCH_P4_MAX_CCCR > X86_PMC_MAX_GENERIC); 1200 BUILD_BUG_ON(ARCH_P4_MAX_CCCR > X86_PMC_MAX_GENERIC);
1202 1201
1203 rdmsr(MSR_IA32_MISC_ENABLE, low, high); 1202 rdmsr(MSR_IA32_MISC_ENABLE, low, high);
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index e2a3f0606da4..f478ff6877ef 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -135,20 +135,6 @@ print_context_stack_bp(struct thread_info *tinfo,
135} 135}
136EXPORT_SYMBOL_GPL(print_context_stack_bp); 136EXPORT_SYMBOL_GPL(print_context_stack_bp);
137 137
138
139static void
140print_trace_warning_symbol(void *data, char *msg, unsigned long symbol)
141{
142 printk(data);
143 print_symbol(msg, symbol);
144 printk("\n");
145}
146
147static void print_trace_warning(void *data, char *msg)
148{
149 printk("%s%s\n", (char *)data, msg);
150}
151
152static int print_trace_stack(void *data, char *name) 138static int print_trace_stack(void *data, char *name)
153{ 139{
154 printk("%s <%s> ", (char *)data, name); 140 printk("%s <%s> ", (char *)data, name);
@@ -166,8 +152,6 @@ static void print_trace_address(void *data, unsigned long addr, int reliable)
166} 152}
167 153
168static const struct stacktrace_ops print_trace_ops = { 154static const struct stacktrace_ops print_trace_ops = {
169 .warning = print_trace_warning,
170 .warning_symbol = print_trace_warning_symbol,
171 .stack = print_trace_stack, 155 .stack = print_trace_stack,
172 .address = print_trace_address, 156 .address = print_trace_address,
173 .walk_stack = print_context_stack, 157 .walk_stack = print_context_stack,
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index ab23f1ad4bf1..52f256f2cc81 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -24,6 +24,7 @@
24#include <linux/bug.h> 24#include <linux/bug.h>
25#include <linux/mm.h> 25#include <linux/mm.h>
26#include <linux/gfp.h> 26#include <linux/gfp.h>
27#include <linux/jump_label.h>
27 28
28#include <asm/system.h> 29#include <asm/system.h>
29#include <asm/page.h> 30#include <asm/page.h>
diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c
index 6515733a289d..55d9bc03f696 100644
--- a/arch/x86/kernel/stacktrace.c
+++ b/arch/x86/kernel/stacktrace.c
@@ -9,15 +9,6 @@
9#include <linux/uaccess.h> 9#include <linux/uaccess.h>
10#include <asm/stacktrace.h> 10#include <asm/stacktrace.h>
11 11
12static void save_stack_warning(void *data, char *msg)
13{
14}
15
16static void
17save_stack_warning_symbol(void *data, char *msg, unsigned long symbol)
18{
19}
20
21static int save_stack_stack(void *data, char *name) 12static int save_stack_stack(void *data, char *name)
22{ 13{
23 return 0; 14 return 0;
@@ -53,16 +44,12 @@ save_stack_address_nosched(void *data, unsigned long addr, int reliable)
53} 44}
54 45
55static const struct stacktrace_ops save_stack_ops = { 46static const struct stacktrace_ops save_stack_ops = {
56 .warning = save_stack_warning,
57 .warning_symbol = save_stack_warning_symbol,
58 .stack = save_stack_stack, 47 .stack = save_stack_stack,
59 .address = save_stack_address, 48 .address = save_stack_address,
60 .walk_stack = print_context_stack, 49 .walk_stack = print_context_stack,
61}; 50};
62 51
63static const struct stacktrace_ops save_stack_ops_nosched = { 52static const struct stacktrace_ops save_stack_ops_nosched = {
64 .warning = save_stack_warning,
65 .warning_symbol = save_stack_warning_symbol,
66 .stack = save_stack_stack, 53 .stack = save_stack_stack,
67 .address = save_stack_address_nosched, 54 .address = save_stack_address_nosched,
68 .walk_stack = print_context_stack, 55 .walk_stack = print_context_stack,
diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S
index aa4326bfb24a..f2145cfa12a6 100644
--- a/arch/x86/lib/clear_page_64.S
+++ b/arch/x86/lib/clear_page_64.S
@@ -1,5 +1,6 @@
1#include <linux/linkage.h> 1#include <linux/linkage.h>
2#include <asm/dwarf2.h> 2#include <asm/dwarf2.h>
3#include <asm/alternative-asm.h>
3 4
4/* 5/*
5 * Zero a page. 6 * Zero a page.
@@ -14,6 +15,15 @@ ENTRY(clear_page_c)
14 CFI_ENDPROC 15 CFI_ENDPROC
15ENDPROC(clear_page_c) 16ENDPROC(clear_page_c)
16 17
18ENTRY(clear_page_c_e)
19 CFI_STARTPROC
20 movl $4096,%ecx
21 xorl %eax,%eax
22 rep stosb
23 ret
24 CFI_ENDPROC
25ENDPROC(clear_page_c_e)
26
17ENTRY(clear_page) 27ENTRY(clear_page)
18 CFI_STARTPROC 28 CFI_STARTPROC
19 xorl %eax,%eax 29 xorl %eax,%eax
@@ -38,21 +48,26 @@ ENTRY(clear_page)
38.Lclear_page_end: 48.Lclear_page_end:
39ENDPROC(clear_page) 49ENDPROC(clear_page)
40 50
41 /* Some CPUs run faster using the string instructions. 51 /*
42 It is also a lot simpler. Use this when possible */ 52 * Some CPUs support enhanced REP MOVSB/STOSB instructions.
53 * It is recommended to use this when possible.
54 * If enhanced REP MOVSB/STOSB is not available, try to use fast string.
55 * Otherwise, use original function.
56 *
57 */
43 58
44#include <asm/cpufeature.h> 59#include <asm/cpufeature.h>
45 60
46 .section .altinstr_replacement,"ax" 61 .section .altinstr_replacement,"ax"
471: .byte 0xeb /* jmp <disp8> */ 621: .byte 0xeb /* jmp <disp8> */
48 .byte (clear_page_c - clear_page) - (2f - 1b) /* offset */ 63 .byte (clear_page_c - clear_page) - (2f - 1b) /* offset */
492: 642: .byte 0xeb /* jmp <disp8> */
65 .byte (clear_page_c_e - clear_page) - (3f - 2b) /* offset */
663:
50 .previous 67 .previous
51 .section .altinstructions,"a" 68 .section .altinstructions,"a"
52 .align 8 69 altinstruction_entry clear_page,1b,X86_FEATURE_REP_GOOD,\
53 .quad clear_page 70 .Lclear_page_end-clear_page, 2b-1b
54 .quad 1b 71 altinstruction_entry clear_page,2b,X86_FEATURE_ERMS, \
55 .word X86_FEATURE_REP_GOOD 72 .Lclear_page_end-clear_page,3b-2b
56 .byte .Lclear_page_end - clear_page
57 .byte 2b - 1b
58 .previous 73 .previous
diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
index 99e482615195..024840266ba0 100644
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -15,23 +15,30 @@
15#include <asm/asm-offsets.h> 15#include <asm/asm-offsets.h>
16#include <asm/thread_info.h> 16#include <asm/thread_info.h>
17#include <asm/cpufeature.h> 17#include <asm/cpufeature.h>
18#include <asm/alternative-asm.h>
18 19
19 .macro ALTERNATIVE_JUMP feature,orig,alt 20/*
21 * By placing feature2 after feature1 in altinstructions section, we logically
22 * implement:
23 * If CPU has feature2, jmp to alt2 is used
24 * else if CPU has feature1, jmp to alt1 is used
25 * else jmp to orig is used.
26 */
27 .macro ALTERNATIVE_JUMP feature1,feature2,orig,alt1,alt2
200: 280:
21 .byte 0xe9 /* 32bit jump */ 29 .byte 0xe9 /* 32bit jump */
22 .long \orig-1f /* by default jump to orig */ 30 .long \orig-1f /* by default jump to orig */
231: 311:
24 .section .altinstr_replacement,"ax" 32 .section .altinstr_replacement,"ax"
252: .byte 0xe9 /* near jump with 32bit immediate */ 332: .byte 0xe9 /* near jump with 32bit immediate */
26 .long \alt-1b /* offset */ /* or alternatively to alt */ 34 .long \alt1-1b /* offset */ /* or alternatively to alt1 */
353: .byte 0xe9 /* near jump with 32bit immediate */
36 .long \alt2-1b /* offset */ /* or alternatively to alt2 */
27 .previous 37 .previous
38
28 .section .altinstructions,"a" 39 .section .altinstructions,"a"
29 .align 8 40 altinstruction_entry 0b,2b,\feature1,5,5
30 .quad 0b 41 altinstruction_entry 0b,3b,\feature2,5,5
31 .quad 2b
32 .word \feature /* when feature is set */
33 .byte 5
34 .byte 5
35 .previous 42 .previous
36 .endm 43 .endm
37 44
@@ -72,8 +79,10 @@ ENTRY(_copy_to_user)
72 addq %rdx,%rcx 79 addq %rdx,%rcx
73 jc bad_to_user 80 jc bad_to_user
74 cmpq TI_addr_limit(%rax),%rcx 81 cmpq TI_addr_limit(%rax),%rcx
75 jae bad_to_user 82 ja bad_to_user
76 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string 83 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
84 copy_user_generic_unrolled,copy_user_generic_string, \
85 copy_user_enhanced_fast_string
77 CFI_ENDPROC 86 CFI_ENDPROC
78ENDPROC(_copy_to_user) 87ENDPROC(_copy_to_user)
79 88
@@ -85,8 +94,10 @@ ENTRY(_copy_from_user)
85 addq %rdx,%rcx 94 addq %rdx,%rcx
86 jc bad_from_user 95 jc bad_from_user
87 cmpq TI_addr_limit(%rax),%rcx 96 cmpq TI_addr_limit(%rax),%rcx
88 jae bad_from_user 97 ja bad_from_user
89 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string 98 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
99 copy_user_generic_unrolled,copy_user_generic_string, \
100 copy_user_enhanced_fast_string
90 CFI_ENDPROC 101 CFI_ENDPROC
91ENDPROC(_copy_from_user) 102ENDPROC(_copy_from_user)
92 103
@@ -255,3 +266,37 @@ ENTRY(copy_user_generic_string)
255 .previous 266 .previous
256 CFI_ENDPROC 267 CFI_ENDPROC
257ENDPROC(copy_user_generic_string) 268ENDPROC(copy_user_generic_string)
269
270/*
271 * Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
272 * It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
273 *
274 * Input:
275 * rdi destination
276 * rsi source
277 * rdx count
278 *
279 * Output:
280 * eax uncopied bytes or 0 if successful.
281 */
282ENTRY(copy_user_enhanced_fast_string)
283 CFI_STARTPROC
284 andl %edx,%edx
285 jz 2f
286 movl %edx,%ecx
2871: rep
288 movsb
2892: xorl %eax,%eax
290 ret
291
292 .section .fixup,"ax"
29312: movl %ecx,%edx /* ecx is zerorest also */
294 jmp copy_user_handle_tail
295 .previous
296
297 .section __ex_table,"a"
298 .align 8
299 .quad 1b,12b
300 .previous
301 CFI_ENDPROC
302ENDPROC(copy_user_enhanced_fast_string)
diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S
index 75ef61e35e38..daab21dae2d1 100644
--- a/arch/x86/lib/memcpy_64.S
+++ b/arch/x86/lib/memcpy_64.S
@@ -4,6 +4,7 @@
4 4
5#include <asm/cpufeature.h> 5#include <asm/cpufeature.h>
6#include <asm/dwarf2.h> 6#include <asm/dwarf2.h>
7#include <asm/alternative-asm.h>
7 8
8/* 9/*
9 * memcpy - Copy a memory block. 10 * memcpy - Copy a memory block.
@@ -37,6 +38,23 @@
37.Lmemcpy_e: 38.Lmemcpy_e:
38 .previous 39 .previous
39 40
41/*
42 * memcpy_c_e() - enhanced fast string memcpy. This is faster and simpler than
43 * memcpy_c. Use memcpy_c_e when possible.
44 *
45 * This gets patched over the unrolled variant (below) via the
46 * alternative instructions framework:
47 */
48 .section .altinstr_replacement, "ax", @progbits
49.Lmemcpy_c_e:
50 movq %rdi, %rax
51
52 movl %edx, %ecx
53 rep movsb
54 ret
55.Lmemcpy_e_e:
56 .previous
57
40ENTRY(__memcpy) 58ENTRY(__memcpy)
41ENTRY(memcpy) 59ENTRY(memcpy)
42 CFI_STARTPROC 60 CFI_STARTPROC
@@ -171,21 +189,22 @@ ENDPROC(memcpy)
171ENDPROC(__memcpy) 189ENDPROC(__memcpy)
172 190
173 /* 191 /*
174 * Some CPUs run faster using the string copy instructions. 192 * Some CPUs are adding enhanced REP MOVSB/STOSB feature
175 * It is also a lot simpler. Use this when possible: 193 * If the feature is supported, memcpy_c_e() is the first choice.
176 */ 194 * If enhanced rep movsb copy is not available, use fast string copy
177 195 * memcpy_c() when possible. This is faster and code is simpler than
178 .section .altinstructions, "a" 196 * original memcpy().
179 .align 8 197 * Otherwise, original memcpy() is used.
180 .quad memcpy 198 * In .altinstructions section, ERMS feature is placed after REG_GOOD
181 .quad .Lmemcpy_c 199 * feature to implement the right patch order.
182 .word X86_FEATURE_REP_GOOD 200 *
183
184 /*
185 * Replace only beginning, memcpy is used to apply alternatives, 201 * Replace only beginning, memcpy is used to apply alternatives,
186 * so it is silly to overwrite itself with nops - reboot is the 202 * so it is silly to overwrite itself with nops - reboot is the
187 * only outcome... 203 * only outcome...
188 */ 204 */
189 .byte .Lmemcpy_e - .Lmemcpy_c 205 .section .altinstructions, "a"
190 .byte .Lmemcpy_e - .Lmemcpy_c 206 altinstruction_entry memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\
207 .Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c
208 altinstruction_entry memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \
209 .Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e
191 .previous 210 .previous
diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S
index 0ecb8433e5a8..d0ec9c2936d7 100644
--- a/arch/x86/lib/memmove_64.S
+++ b/arch/x86/lib/memmove_64.S
@@ -8,6 +8,7 @@
8#define _STRING_C 8#define _STRING_C
9#include <linux/linkage.h> 9#include <linux/linkage.h>
10#include <asm/dwarf2.h> 10#include <asm/dwarf2.h>
11#include <asm/cpufeature.h>
11 12
12#undef memmove 13#undef memmove
13 14
@@ -24,6 +25,7 @@
24 */ 25 */
25ENTRY(memmove) 26ENTRY(memmove)
26 CFI_STARTPROC 27 CFI_STARTPROC
28
27 /* Handle more 32bytes in loop */ 29 /* Handle more 32bytes in loop */
28 mov %rdi, %rax 30 mov %rdi, %rax
29 cmp $0x20, %rdx 31 cmp $0x20, %rdx
@@ -31,8 +33,13 @@ ENTRY(memmove)
31 33
32 /* Decide forward/backward copy mode */ 34 /* Decide forward/backward copy mode */
33 cmp %rdi, %rsi 35 cmp %rdi, %rsi
34 jb 2f 36 jge .Lmemmove_begin_forward
37 mov %rsi, %r8
38 add %rdx, %r8
39 cmp %rdi, %r8
40 jg 2f
35 41
42.Lmemmove_begin_forward:
36 /* 43 /*
37 * movsq instruction have many startup latency 44 * movsq instruction have many startup latency
38 * so we handle small size by general register. 45 * so we handle small size by general register.
@@ -78,6 +85,8 @@ ENTRY(memmove)
78 rep movsq 85 rep movsq
79 movq %r11, (%r10) 86 movq %r11, (%r10)
80 jmp 13f 87 jmp 13f
88.Lmemmove_end_forward:
89
81 /* 90 /*
82 * Handle data backward by movsq. 91 * Handle data backward by movsq.
83 */ 92 */
@@ -194,4 +203,22 @@ ENTRY(memmove)
19413: 20313:
195 retq 204 retq
196 CFI_ENDPROC 205 CFI_ENDPROC
206
207 .section .altinstr_replacement,"ax"
208.Lmemmove_begin_forward_efs:
209 /* Forward moving data. */
210 movq %rdx, %rcx
211 rep movsb
212 retq
213.Lmemmove_end_forward_efs:
214 .previous
215
216 .section .altinstructions,"a"
217 .align 8
218 .quad .Lmemmove_begin_forward
219 .quad .Lmemmove_begin_forward_efs
220 .word X86_FEATURE_ERMS
221 .byte .Lmemmove_end_forward-.Lmemmove_begin_forward
222 .byte .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs
223 .previous
197ENDPROC(memmove) 224ENDPROC(memmove)
diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S
index 09d344269652..79bd454b78a3 100644
--- a/arch/x86/lib/memset_64.S
+++ b/arch/x86/lib/memset_64.S
@@ -2,9 +2,13 @@
2 2
3#include <linux/linkage.h> 3#include <linux/linkage.h>
4#include <asm/dwarf2.h> 4#include <asm/dwarf2.h>
5#include <asm/cpufeature.h>
6#include <asm/alternative-asm.h>
5 7
6/* 8/*
7 * ISO C memset - set a memory block to a byte value. 9 * ISO C memset - set a memory block to a byte value. This function uses fast
10 * string to get better performance than the original function. The code is
11 * simpler and shorter than the orignal function as well.
8 * 12 *
9 * rdi destination 13 * rdi destination
10 * rsi value (char) 14 * rsi value (char)
@@ -31,6 +35,28 @@
31.Lmemset_e: 35.Lmemset_e:
32 .previous 36 .previous
33 37
38/*
39 * ISO C memset - set a memory block to a byte value. This function uses
40 * enhanced rep stosb to override the fast string function.
41 * The code is simpler and shorter than the fast string function as well.
42 *
43 * rdi destination
44 * rsi value (char)
45 * rdx count (bytes)
46 *
47 * rax original destination
48 */
49 .section .altinstr_replacement, "ax", @progbits
50.Lmemset_c_e:
51 movq %rdi,%r9
52 movb %sil,%al
53 movl %edx,%ecx
54 rep stosb
55 movq %r9,%rax
56 ret
57.Lmemset_e_e:
58 .previous
59
34ENTRY(memset) 60ENTRY(memset)
35ENTRY(__memset) 61ENTRY(__memset)
36 CFI_STARTPROC 62 CFI_STARTPROC
@@ -112,16 +138,20 @@ ENTRY(__memset)
112ENDPROC(memset) 138ENDPROC(memset)
113ENDPROC(__memset) 139ENDPROC(__memset)
114 140
115 /* Some CPUs run faster using the string instructions. 141 /* Some CPUs support enhanced REP MOVSB/STOSB feature.
116 It is also a lot simpler. Use this when possible */ 142 * It is recommended to use this when possible.
117 143 *
118#include <asm/cpufeature.h> 144 * If enhanced REP MOVSB/STOSB feature is not available, use fast string
119 145 * instructions.
146 *
147 * Otherwise, use original memset function.
148 *
149 * In .altinstructions section, ERMS feature is placed after REG_GOOD
150 * feature to implement the right patch order.
151 */
120 .section .altinstructions,"a" 152 .section .altinstructions,"a"
121 .align 8 153 altinstruction_entry memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\
122 .quad memset 154 .Lfinal-memset,.Lmemset_e-.Lmemset_c
123 .quad .Lmemset_c 155 altinstruction_entry memset,.Lmemset_c_e,X86_FEATURE_ERMS, \
124 .word X86_FEATURE_REP_GOOD 156 .Lfinal-memset,.Lmemset_e_e-.Lmemset_c_e
125 .byte .Lfinal - memset
126 .byte .Lmemset_e - .Lmemset_c
127 .previous 157 .previous
diff --git a/arch/x86/oprofile/backtrace.c b/arch/x86/oprofile/backtrace.c
index 2d49d4e19a36..a5b64ab4cd6e 100644
--- a/arch/x86/oprofile/backtrace.c
+++ b/arch/x86/oprofile/backtrace.c
@@ -16,17 +16,6 @@
16#include <asm/stacktrace.h> 16#include <asm/stacktrace.h>
17#include <linux/compat.h> 17#include <linux/compat.h>
18 18
19static void backtrace_warning_symbol(void *data, char *msg,
20 unsigned long symbol)
21{
22 /* Ignore warnings */
23}
24
25static void backtrace_warning(void *data, char *msg)
26{
27 /* Ignore warnings */
28}
29
30static int backtrace_stack(void *data, char *name) 19static int backtrace_stack(void *data, char *name)
31{ 20{
32 /* Yes, we want all stacks */ 21 /* Yes, we want all stacks */
@@ -42,8 +31,6 @@ static void backtrace_address(void *data, unsigned long addr, int reliable)
42} 31}
43 32
44static struct stacktrace_ops backtrace_ops = { 33static struct stacktrace_ops backtrace_ops = {
45 .warning = backtrace_warning,
46 .warning_symbol = backtrace_warning_symbol,
47 .stack = backtrace_stack, 34 .stack = backtrace_stack,
48 .address = backtrace_address, 35 .address = backtrace_address,
49 .walk_stack = print_context_stack, 36 .walk_stack = print_context_stack,
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index b27445e00b6c..077c00d94f6e 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -170,6 +170,10 @@
170 STRUCT_ALIGN(); \ 170 STRUCT_ALIGN(); \
171 *(__tracepoints) \ 171 *(__tracepoints) \
172 /* implement dynamic printk debug */ \ 172 /* implement dynamic printk debug */ \
173 . = ALIGN(8); \
174 VMLINUX_SYMBOL(__start___jump_table) = .; \
175 *(__jump_table) \
176 VMLINUX_SYMBOL(__stop___jump_table) = .; \
173 . = ALIGN(8); \ 177 . = ALIGN(8); \
174 VMLINUX_SYMBOL(__start___verbose) = .; \ 178 VMLINUX_SYMBOL(__start___verbose) = .; \
175 *(__verbose) \ 179 *(__verbose) \
@@ -228,8 +232,6 @@
228 \ 232 \
229 BUG_TABLE \ 233 BUG_TABLE \
230 \ 234 \
231 JUMP_TABLE \
232 \
233 /* PCI quirks */ \ 235 /* PCI quirks */ \
234 .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ 236 .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \
235 VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \ 237 VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \
@@ -589,14 +591,6 @@
589#define BUG_TABLE 591#define BUG_TABLE
590#endif 592#endif
591 593
592#define JUMP_TABLE \
593 . = ALIGN(8); \
594 __jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) { \
595 VMLINUX_SYMBOL(__start___jump_table) = .; \
596 *(__jump_table) \
597 VMLINUX_SYMBOL(__stop___jump_table) = .; \
598 }
599
600#ifdef CONFIG_PM_TRACE 594#ifdef CONFIG_PM_TRACE
601#define TRACEDATA \ 595#define TRACEDATA \
602 . = ALIGN(4); \ 596 . = ALIGN(4); \
diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h
index 0c9653f11c18..e747ecd48e1c 100644
--- a/include/linux/dynamic_debug.h
+++ b/include/linux/dynamic_debug.h
@@ -1,8 +1,6 @@
1#ifndef _DYNAMIC_DEBUG_H 1#ifndef _DYNAMIC_DEBUG_H
2#define _DYNAMIC_DEBUG_H 2#define _DYNAMIC_DEBUG_H
3 3
4#include <linux/jump_label.h>
5
6/* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which 4/* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which
7 * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They 5 * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They
8 * use independent hash functions, to reduce the chance of false positives. 6 * use independent hash functions, to reduce the chance of false positives.
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index ca29e03c1fac..9d88e1cb5dbb 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -29,9 +29,22 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
29 29
30typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip); 30typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip);
31 31
32struct ftrace_hash;
33
34enum {
35 FTRACE_OPS_FL_ENABLED = 1 << 0,
36 FTRACE_OPS_FL_GLOBAL = 1 << 1,
37 FTRACE_OPS_FL_DYNAMIC = 1 << 2,
38};
39
32struct ftrace_ops { 40struct ftrace_ops {
33 ftrace_func_t func; 41 ftrace_func_t func;
34 struct ftrace_ops *next; 42 struct ftrace_ops *next;
43 unsigned long flags;
44#ifdef CONFIG_DYNAMIC_FTRACE
45 struct ftrace_hash *notrace_hash;
46 struct ftrace_hash *filter_hash;
47#endif
35}; 48};
36 49
37extern int function_trace_stop; 50extern int function_trace_stop;
@@ -146,14 +159,13 @@ extern void unregister_ftrace_function_probe_all(char *glob);
146extern int ftrace_text_reserved(void *start, void *end); 159extern int ftrace_text_reserved(void *start, void *end);
147 160
148enum { 161enum {
149 FTRACE_FL_FREE = (1 << 0), 162 FTRACE_FL_ENABLED = (1 << 30),
150 FTRACE_FL_FAILED = (1 << 1), 163 FTRACE_FL_FREE = (1 << 31),
151 FTRACE_FL_FILTER = (1 << 2),
152 FTRACE_FL_ENABLED = (1 << 3),
153 FTRACE_FL_NOTRACE = (1 << 4),
154 FTRACE_FL_CONVERTED = (1 << 5),
155}; 164};
156 165
166#define FTRACE_FL_MASK (0x3UL << 30)
167#define FTRACE_REF_MAX ((1 << 30) - 1)
168
157struct dyn_ftrace { 169struct dyn_ftrace {
158 union { 170 union {
159 unsigned long ip; /* address of mcount call-site */ 171 unsigned long ip; /* address of mcount call-site */
@@ -167,7 +179,12 @@ struct dyn_ftrace {
167}; 179};
168 180
169int ftrace_force_update(void); 181int ftrace_force_update(void);
170void ftrace_set_filter(unsigned char *buf, int len, int reset); 182void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
183 int len, int reset);
184void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
185 int len, int reset);
186void ftrace_set_global_filter(unsigned char *buf, int len, int reset);
187void ftrace_set_global_notrace(unsigned char *buf, int len, int reset);
171 188
172int register_ftrace_command(struct ftrace_func_command *cmd); 189int register_ftrace_command(struct ftrace_func_command *cmd);
173int unregister_ftrace_command(struct ftrace_func_command *cmd); 190int unregister_ftrace_command(struct ftrace_func_command *cmd);
diff --git a/include/linux/init.h b/include/linux/init.h
index 577671c55153..9146f39cdddf 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -79,29 +79,29 @@
79#define __exitused __used 79#define __exitused __used
80#endif 80#endif
81 81
82#define __exit __section(.exit.text) __exitused __cold 82#define __exit __section(.exit.text) __exitused __cold notrace
83 83
84/* Used for HOTPLUG */ 84/* Used for HOTPLUG */
85#define __devinit __section(.devinit.text) __cold 85#define __devinit __section(.devinit.text) __cold notrace
86#define __devinitdata __section(.devinit.data) 86#define __devinitdata __section(.devinit.data)
87#define __devinitconst __section(.devinit.rodata) 87#define __devinitconst __section(.devinit.rodata)
88#define __devexit __section(.devexit.text) __exitused __cold 88#define __devexit __section(.devexit.text) __exitused __cold notrace
89#define __devexitdata __section(.devexit.data) 89#define __devexitdata __section(.devexit.data)
90#define __devexitconst __section(.devexit.rodata) 90#define __devexitconst __section(.devexit.rodata)
91 91
92/* Used for HOTPLUG_CPU */ 92/* Used for HOTPLUG_CPU */
93#define __cpuinit __section(.cpuinit.text) __cold 93#define __cpuinit __section(.cpuinit.text) __cold notrace
94#define __cpuinitdata __section(.cpuinit.data) 94#define __cpuinitdata __section(.cpuinit.data)
95#define __cpuinitconst __section(.cpuinit.rodata) 95#define __cpuinitconst __section(.cpuinit.rodata)
96#define __cpuexit __section(.cpuexit.text) __exitused __cold 96#define __cpuexit __section(.cpuexit.text) __exitused __cold notrace
97#define __cpuexitdata __section(.cpuexit.data) 97#define __cpuexitdata __section(.cpuexit.data)
98#define __cpuexitconst __section(.cpuexit.rodata) 98#define __cpuexitconst __section(.cpuexit.rodata)
99 99
100/* Used for MEMORY_HOTPLUG */ 100/* Used for MEMORY_HOTPLUG */
101#define __meminit __section(.meminit.text) __cold 101#define __meminit __section(.meminit.text) __cold notrace
102#define __meminitdata __section(.meminit.data) 102#define __meminitdata __section(.meminit.data)
103#define __meminitconst __section(.meminit.rodata) 103#define __meminitconst __section(.meminit.rodata)
104#define __memexit __section(.memexit.text) __exitused __cold 104#define __memexit __section(.memexit.text) __exitused __cold notrace
105#define __memexitdata __section(.memexit.data) 105#define __memexitdata __section(.memexit.data)
106#define __memexitconst __section(.memexit.rodata) 106#define __memexitconst __section(.memexit.rodata)
107 107
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 7880f18e4b86..83e745f3ead7 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -1,20 +1,43 @@
1#ifndef _LINUX_JUMP_LABEL_H 1#ifndef _LINUX_JUMP_LABEL_H
2#define _LINUX_JUMP_LABEL_H 2#define _LINUX_JUMP_LABEL_H
3 3
4#include <linux/types.h>
5#include <linux/compiler.h>
6
4#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 7#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
8
9struct jump_label_key {
10 atomic_t enabled;
11 struct jump_entry *entries;
12#ifdef CONFIG_MODULES
13 struct jump_label_mod *next;
14#endif
15};
16
5# include <asm/jump_label.h> 17# include <asm/jump_label.h>
6# define HAVE_JUMP_LABEL 18# define HAVE_JUMP_LABEL
7#endif 19#endif
8 20
9enum jump_label_type { 21enum jump_label_type {
22 JUMP_LABEL_DISABLE = 0,
10 JUMP_LABEL_ENABLE, 23 JUMP_LABEL_ENABLE,
11 JUMP_LABEL_DISABLE
12}; 24};
13 25
14struct module; 26struct module;
15 27
16#ifdef HAVE_JUMP_LABEL 28#ifdef HAVE_JUMP_LABEL
17 29
30#ifdef CONFIG_MODULES
31#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
32#else
33#define JUMP_LABEL_INIT {{ 0 }, NULL}
34#endif
35
36static __always_inline bool static_branch(struct jump_label_key *key)
37{
38 return arch_static_branch(key);
39}
40
18extern struct jump_entry __start___jump_table[]; 41extern struct jump_entry __start___jump_table[];
19extern struct jump_entry __stop___jump_table[]; 42extern struct jump_entry __stop___jump_table[];
20 43
@@ -23,37 +46,37 @@ extern void jump_label_unlock(void);
23extern void arch_jump_label_transform(struct jump_entry *entry, 46extern void arch_jump_label_transform(struct jump_entry *entry,
24 enum jump_label_type type); 47 enum jump_label_type type);
25extern void arch_jump_label_text_poke_early(jump_label_t addr); 48extern void arch_jump_label_text_poke_early(jump_label_t addr);
26extern void jump_label_update(unsigned long key, enum jump_label_type type);
27extern void jump_label_apply_nops(struct module *mod);
28extern int jump_label_text_reserved(void *start, void *end); 49extern int jump_label_text_reserved(void *start, void *end);
50extern void jump_label_inc(struct jump_label_key *key);
51extern void jump_label_dec(struct jump_label_key *key);
52extern bool jump_label_enabled(struct jump_label_key *key);
53extern void jump_label_apply_nops(struct module *mod);
29 54
30#define jump_label_enable(key) \ 55#else
31 jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
32 56
33#define jump_label_disable(key) \ 57#include <asm/atomic.h>
34 jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
35 58
36#else 59#define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
37 60
38#define JUMP_LABEL(key, label) \ 61struct jump_label_key {
39do { \ 62 atomic_t enabled;
40 if (unlikely(*key)) \ 63};
41 goto label; \
42} while (0)
43 64
44#define jump_label_enable(cond_var) \ 65static __always_inline bool static_branch(struct jump_label_key *key)
45do { \ 66{
46 *(cond_var) = 1; \ 67 if (unlikely(atomic_read(&key->enabled)))
47} while (0) 68 return true;
69 return false;
70}
48 71
49#define jump_label_disable(cond_var) \ 72static inline void jump_label_inc(struct jump_label_key *key)
50do { \ 73{
51 *(cond_var) = 0; \ 74 atomic_inc(&key->enabled);
52} while (0) 75}
53 76
54static inline int jump_label_apply_nops(struct module *mod) 77static inline void jump_label_dec(struct jump_label_key *key)
55{ 78{
56 return 0; 79 atomic_dec(&key->enabled);
57} 80}
58 81
59static inline int jump_label_text_reserved(void *start, void *end) 82static inline int jump_label_text_reserved(void *start, void *end)
@@ -64,16 +87,16 @@ static inline int jump_label_text_reserved(void *start, void *end)
64static inline void jump_label_lock(void) {} 87static inline void jump_label_lock(void) {}
65static inline void jump_label_unlock(void) {} 88static inline void jump_label_unlock(void) {}
66 89
67#endif 90static inline bool jump_label_enabled(struct jump_label_key *key)
91{
92 return !!atomic_read(&key->enabled);
93}
68 94
69#define COND_STMT(key, stmt) \ 95static inline int jump_label_apply_nops(struct module *mod)
70do { \ 96{
71 __label__ jl_enabled; \ 97 return 0;
72 JUMP_LABEL(key, jl_enabled); \ 98}
73 if (0) { \ 99
74jl_enabled: \ 100#endif
75 stmt; \
76 } \
77} while (0)
78 101
79#endif 102#endif
diff --git a/include/linux/jump_label_ref.h b/include/linux/jump_label_ref.h
deleted file mode 100644
index e5d012ad92c6..000000000000
--- a/include/linux/jump_label_ref.h
+++ /dev/null
@@ -1,44 +0,0 @@
1#ifndef _LINUX_JUMP_LABEL_REF_H
2#define _LINUX_JUMP_LABEL_REF_H
3
4#include <linux/jump_label.h>
5#include <asm/atomic.h>
6
7#ifdef HAVE_JUMP_LABEL
8
9static inline void jump_label_inc(atomic_t *key)
10{
11 if (atomic_add_return(1, key) == 1)
12 jump_label_enable(key);
13}
14
15static inline void jump_label_dec(atomic_t *key)
16{
17 if (atomic_dec_and_test(key))
18 jump_label_disable(key);
19}
20
21#else /* !HAVE_JUMP_LABEL */
22
23static inline void jump_label_inc(atomic_t *key)
24{
25 atomic_inc(key);
26}
27
28static inline void jump_label_dec(atomic_t *key)
29{
30 atomic_dec(key);
31}
32
33#undef JUMP_LABEL
34#define JUMP_LABEL(key, label) \
35do { \
36 if (unlikely(__builtin_choose_expr( \
37 __builtin_types_compatible_p(typeof(key), atomic_t *), \
38 atomic_read((atomic_t *)(key)), *(key)))) \
39 goto label; \
40} while (0)
41
42#endif /* HAVE_JUMP_LABEL */
43
44#endif /* _LINUX_JUMP_LABEL_REF_H */
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 00cec4dc0ae2..f37ba716ef8b 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -283,6 +283,7 @@ extern char *get_options(const char *str, int nints, int *ints);
283extern unsigned long long memparse(const char *ptr, char **retptr); 283extern unsigned long long memparse(const char *ptr, char **retptr);
284 284
285extern int core_kernel_text(unsigned long addr); 285extern int core_kernel_text(unsigned long addr);
286extern int core_kernel_data(unsigned long addr);
286extern int __kernel_text_address(unsigned long addr); 287extern int __kernel_text_address(unsigned long addr);
287extern int kernel_text_address(unsigned long addr); 288extern int kernel_text_address(unsigned long addr);
288extern int func_ptr_is_kernel_text(void *ptr); 289extern int func_ptr_is_kernel_text(void *ptr);
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index ee9f1e782800..3412684ce5d5 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -2,8 +2,8 @@
2 * Performance events: 2 * Performance events:
3 * 3 *
4 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de> 4 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009, Red Hat, Inc., Ingo Molnar 5 * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2009, Red Hat, Inc., Peter Zijlstra 6 * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
7 * 7 *
8 * Data type definitions, declarations, prototypes. 8 * Data type definitions, declarations, prototypes.
9 * 9 *
@@ -52,6 +52,8 @@ enum perf_hw_id {
52 PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4, 52 PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4,
53 PERF_COUNT_HW_BRANCH_MISSES = 5, 53 PERF_COUNT_HW_BRANCH_MISSES = 5,
54 PERF_COUNT_HW_BUS_CYCLES = 6, 54 PERF_COUNT_HW_BUS_CYCLES = 6,
55 PERF_COUNT_HW_STALLED_CYCLES_FRONTEND = 7,
56 PERF_COUNT_HW_STALLED_CYCLES_BACKEND = 8,
55 57
56 PERF_COUNT_HW_MAX, /* non-ABI */ 58 PERF_COUNT_HW_MAX, /* non-ABI */
57}; 59};
@@ -468,9 +470,9 @@ enum perf_callchain_context {
468 PERF_CONTEXT_MAX = (__u64)-4095, 470 PERF_CONTEXT_MAX = (__u64)-4095,
469}; 471};
470 472
471#define PERF_FLAG_FD_NO_GROUP (1U << 0) 473#define PERF_FLAG_FD_NO_GROUP (1U << 0)
472#define PERF_FLAG_FD_OUTPUT (1U << 1) 474#define PERF_FLAG_FD_OUTPUT (1U << 1)
473#define PERF_FLAG_PID_CGROUP (1U << 2) /* pid=cgroup id, per-cpu mode only */ 475#define PERF_FLAG_PID_CGROUP (1U << 2) /* pid=cgroup id, per-cpu mode only */
474 476
475#ifdef __KERNEL__ 477#ifdef __KERNEL__
476/* 478/*
@@ -484,9 +486,9 @@ enum perf_callchain_context {
484#endif 486#endif
485 487
486struct perf_guest_info_callbacks { 488struct perf_guest_info_callbacks {
487 int (*is_in_guest) (void); 489 int (*is_in_guest)(void);
488 int (*is_user_mode) (void); 490 int (*is_user_mode)(void);
489 unsigned long (*get_guest_ip) (void); 491 unsigned long (*get_guest_ip)(void);
490}; 492};
491 493
492#ifdef CONFIG_HAVE_HW_BREAKPOINT 494#ifdef CONFIG_HAVE_HW_BREAKPOINT
@@ -505,7 +507,7 @@ struct perf_guest_info_callbacks {
505#include <linux/ftrace.h> 507#include <linux/ftrace.h>
506#include <linux/cpu.h> 508#include <linux/cpu.h>
507#include <linux/irq_work.h> 509#include <linux/irq_work.h>
508#include <linux/jump_label_ref.h> 510#include <linux/jump_label.h>
509#include <asm/atomic.h> 511#include <asm/atomic.h>
510#include <asm/local.h> 512#include <asm/local.h>
511 513
@@ -652,19 +654,19 @@ struct pmu {
652 * Start the transaction, after this ->add() doesn't need to 654 * Start the transaction, after this ->add() doesn't need to
653 * do schedulability tests. 655 * do schedulability tests.
654 */ 656 */
655 void (*start_txn) (struct pmu *pmu); /* optional */ 657 void (*start_txn) (struct pmu *pmu); /* optional */
656 /* 658 /*
657 * If ->start_txn() disabled the ->add() schedulability test 659 * If ->start_txn() disabled the ->add() schedulability test
658 * then ->commit_txn() is required to perform one. On success 660 * then ->commit_txn() is required to perform one. On success
659 * the transaction is closed. On error the transaction is kept 661 * the transaction is closed. On error the transaction is kept
660 * open until ->cancel_txn() is called. 662 * open until ->cancel_txn() is called.
661 */ 663 */
662 int (*commit_txn) (struct pmu *pmu); /* optional */ 664 int (*commit_txn) (struct pmu *pmu); /* optional */
663 /* 665 /*
664 * Will cancel the transaction, assumes ->del() is called 666 * Will cancel the transaction, assumes ->del() is called
665 * for each successful ->add() during the transaction. 667 * for each successful ->add() during the transaction.
666 */ 668 */
667 void (*cancel_txn) (struct pmu *pmu); /* optional */ 669 void (*cancel_txn) (struct pmu *pmu); /* optional */
668}; 670};
669 671
670/** 672/**
@@ -712,15 +714,15 @@ typedef void (*perf_overflow_handler_t)(struct perf_event *, int,
712 struct pt_regs *regs); 714 struct pt_regs *regs);
713 715
714enum perf_group_flag { 716enum perf_group_flag {
715 PERF_GROUP_SOFTWARE = 0x1, 717 PERF_GROUP_SOFTWARE = 0x1,
716}; 718};
717 719
718#define SWEVENT_HLIST_BITS 8 720#define SWEVENT_HLIST_BITS 8
719#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS) 721#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
720 722
721struct swevent_hlist { 723struct swevent_hlist {
722 struct hlist_head heads[SWEVENT_HLIST_SIZE]; 724 struct hlist_head heads[SWEVENT_HLIST_SIZE];
723 struct rcu_head rcu_head; 725 struct rcu_head rcu_head;
724}; 726};
725 727
726#define PERF_ATTACH_CONTEXT 0x01 728#define PERF_ATTACH_CONTEXT 0x01
@@ -733,13 +735,13 @@ struct swevent_hlist {
733 * This is a per-cpu dynamically allocated data structure. 735 * This is a per-cpu dynamically allocated data structure.
734 */ 736 */
735struct perf_cgroup_info { 737struct perf_cgroup_info {
736 u64 time; 738 u64 time;
737 u64 timestamp; 739 u64 timestamp;
738}; 740};
739 741
740struct perf_cgroup { 742struct perf_cgroup {
741 struct cgroup_subsys_state css; 743 struct cgroup_subsys_state css;
742 struct perf_cgroup_info *info; /* timing info, one per cpu */ 744 struct perf_cgroup_info *info; /* timing info, one per cpu */
743}; 745};
744#endif 746#endif
745 747
@@ -923,7 +925,7 @@ struct perf_event_context {
923 925
924/* 926/*
925 * Number of contexts where an event can trigger: 927 * Number of contexts where an event can trigger:
926 * task, softirq, hardirq, nmi. 928 * task, softirq, hardirq, nmi.
927 */ 929 */
928#define PERF_NR_CONTEXTS 4 930#define PERF_NR_CONTEXTS 4
929 931
@@ -1001,8 +1003,7 @@ struct perf_sample_data {
1001 struct perf_raw_record *raw; 1003 struct perf_raw_record *raw;
1002}; 1004};
1003 1005
1004static inline 1006static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
1005void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
1006{ 1007{
1007 data->addr = addr; 1008 data->addr = addr;
1008 data->raw = NULL; 1009 data->raw = NULL;
@@ -1034,13 +1035,12 @@ static inline int is_software_event(struct perf_event *event)
1034 return event->pmu->task_ctx_nr == perf_sw_context; 1035 return event->pmu->task_ctx_nr == perf_sw_context;
1035} 1036}
1036 1037
1037extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; 1038extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
1038 1039
1039extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64); 1040extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64);
1040 1041
1041#ifndef perf_arch_fetch_caller_regs 1042#ifndef perf_arch_fetch_caller_regs
1042static inline void 1043static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
1043perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
1044#endif 1044#endif
1045 1045
1046/* 1046/*
@@ -1063,26 +1063,24 @@ perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
1063{ 1063{
1064 struct pt_regs hot_regs; 1064 struct pt_regs hot_regs;
1065 1065
1066 JUMP_LABEL(&perf_swevent_enabled[event_id], have_event); 1066 if (static_branch(&perf_swevent_enabled[event_id])) {
1067 return; 1067 if (!regs) {
1068 1068 perf_fetch_caller_regs(&hot_regs);
1069have_event: 1069 regs = &hot_regs;
1070 if (!regs) { 1070 }
1071 perf_fetch_caller_regs(&hot_regs); 1071 __perf_sw_event(event_id, nr, nmi, regs, addr);
1072 regs = &hot_regs;
1073 } 1072 }
1074 __perf_sw_event(event_id, nr, nmi, regs, addr);
1075} 1073}
1076 1074
1077extern atomic_t perf_sched_events; 1075extern struct jump_label_key perf_sched_events;
1078 1076
1079static inline void perf_event_task_sched_in(struct task_struct *task) 1077static inline void perf_event_task_sched_in(struct task_struct *task)
1080{ 1078{
1081 COND_STMT(&perf_sched_events, __perf_event_task_sched_in(task)); 1079 if (static_branch(&perf_sched_events))
1080 __perf_event_task_sched_in(task);
1082} 1081}
1083 1082
1084static inline 1083static inline void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
1085void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
1086{ 1084{
1087 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0); 1085 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
1088 1086
@@ -1100,14 +1098,10 @@ extern void perf_event_fork(struct task_struct *tsk);
1100/* Callchains */ 1098/* Callchains */
1101DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); 1099DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
1102 1100
1103extern void perf_callchain_user(struct perf_callchain_entry *entry, 1101extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
1104 struct pt_regs *regs); 1102extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
1105extern void perf_callchain_kernel(struct perf_callchain_entry *entry,
1106 struct pt_regs *regs);
1107
1108 1103
1109static inline void 1104static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
1110perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
1111{ 1105{
1112 if (entry->nr < PERF_MAX_STACK_DEPTH) 1106 if (entry->nr < PERF_MAX_STACK_DEPTH)
1113 entry->ip[entry->nr++] = ip; 1107 entry->ip[entry->nr++] = ip;
@@ -1143,9 +1137,9 @@ extern void perf_tp_event(u64 addr, u64 count, void *record,
1143extern void perf_bp_event(struct perf_event *event, void *data); 1137extern void perf_bp_event(struct perf_event *event, void *data);
1144 1138
1145#ifndef perf_misc_flags 1139#ifndef perf_misc_flags
1146#define perf_misc_flags(regs) (user_mode(regs) ? PERF_RECORD_MISC_USER : \ 1140# define perf_misc_flags(regs) \
1147 PERF_RECORD_MISC_KERNEL) 1141 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
1148#define perf_instruction_pointer(regs) instruction_pointer(regs) 1142# define perf_instruction_pointer(regs) instruction_pointer(regs)
1149#endif 1143#endif
1150 1144
1151extern int perf_output_begin(struct perf_output_handle *handle, 1145extern int perf_output_begin(struct perf_output_handle *handle,
@@ -1180,9 +1174,9 @@ static inline void
1180perf_bp_event(struct perf_event *event, void *data) { } 1174perf_bp_event(struct perf_event *event, void *data) { }
1181 1175
1182static inline int perf_register_guest_info_callbacks 1176static inline int perf_register_guest_info_callbacks
1183(struct perf_guest_info_callbacks *callbacks) { return 0; } 1177(struct perf_guest_info_callbacks *callbacks) { return 0; }
1184static inline int perf_unregister_guest_info_callbacks 1178static inline int perf_unregister_guest_info_callbacks
1185(struct perf_guest_info_callbacks *callbacks) { return 0; } 1179(struct perf_guest_info_callbacks *callbacks) { return 0; }
1186 1180
1187static inline void perf_event_mmap(struct vm_area_struct *vma) { } 1181static inline void perf_event_mmap(struct vm_area_struct *vma) { }
1188static inline void perf_event_comm(struct task_struct *tsk) { } 1182static inline void perf_event_comm(struct task_struct *tsk) { }
@@ -1195,23 +1189,22 @@ static inline void perf_event_disable(struct perf_event *event) { }
1195static inline void perf_event_task_tick(void) { } 1189static inline void perf_event_task_tick(void) { }
1196#endif 1190#endif
1197 1191
1198#define perf_output_put(handle, x) \ 1192#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1199 perf_output_copy((handle), &(x), sizeof(x))
1200 1193
1201/* 1194/*
1202 * This has to have a higher priority than migration_notifier in sched.c. 1195 * This has to have a higher priority than migration_notifier in sched.c.
1203 */ 1196 */
1204#define perf_cpu_notifier(fn) \ 1197#define perf_cpu_notifier(fn) \
1205do { \ 1198do { \
1206 static struct notifier_block fn##_nb __cpuinitdata = \ 1199 static struct notifier_block fn##_nb __cpuinitdata = \
1207 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ 1200 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
1208 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ 1201 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \
1209 (void *)(unsigned long)smp_processor_id()); \ 1202 (void *)(unsigned long)smp_processor_id()); \
1210 fn(&fn##_nb, (unsigned long)CPU_STARTING, \ 1203 fn(&fn##_nb, (unsigned long)CPU_STARTING, \
1211 (void *)(unsigned long)smp_processor_id()); \ 1204 (void *)(unsigned long)smp_processor_id()); \
1212 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \ 1205 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \
1213 (void *)(unsigned long)smp_processor_id()); \ 1206 (void *)(unsigned long)smp_processor_id()); \
1214 register_cpu_notifier(&fn##_nb); \ 1207 register_cpu_notifier(&fn##_nb); \
1215} while (0) 1208} while (0)
1216 1209
1217#endif /* __KERNEL__ */ 1210#endif /* __KERNEL__ */
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 97c84a58efb8..d530a4460a0b 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -29,7 +29,7 @@ struct tracepoint_func {
29 29
30struct tracepoint { 30struct tracepoint {
31 const char *name; /* Tracepoint name */ 31 const char *name; /* Tracepoint name */
32 int state; /* State. */ 32 struct jump_label_key key;
33 void (*regfunc)(void); 33 void (*regfunc)(void);
34 void (*unregfunc)(void); 34 void (*unregfunc)(void);
35 struct tracepoint_func __rcu *funcs; 35 struct tracepoint_func __rcu *funcs;
@@ -146,9 +146,7 @@ void tracepoint_update_probe_range(struct tracepoint * const *begin,
146 extern struct tracepoint __tracepoint_##name; \ 146 extern struct tracepoint __tracepoint_##name; \
147 static inline void trace_##name(proto) \ 147 static inline void trace_##name(proto) \
148 { \ 148 { \
149 JUMP_LABEL(&__tracepoint_##name.state, do_trace); \ 149 if (static_branch(&__tracepoint_##name.key)) \
150 return; \
151do_trace: \
152 __DO_TRACE(&__tracepoint_##name, \ 150 __DO_TRACE(&__tracepoint_##name, \
153 TP_PROTO(data_proto), \ 151 TP_PROTO(data_proto), \
154 TP_ARGS(data_args), \ 152 TP_ARGS(data_args), \
@@ -176,14 +174,14 @@ do_trace: \
176 * structures, so we create an array of pointers that will be used for iteration 174 * structures, so we create an array of pointers that will be used for iteration
177 * on the tracepoints. 175 * on the tracepoints.
178 */ 176 */
179#define DEFINE_TRACE_FN(name, reg, unreg) \ 177#define DEFINE_TRACE_FN(name, reg, unreg) \
180 static const char __tpstrtab_##name[] \ 178 static const char __tpstrtab_##name[] \
181 __attribute__((section("__tracepoints_strings"))) = #name; \ 179 __attribute__((section("__tracepoints_strings"))) = #name; \
182 struct tracepoint __tracepoint_##name \ 180 struct tracepoint __tracepoint_##name \
183 __attribute__((section("__tracepoints"))) = \ 181 __attribute__((section("__tracepoints"))) = \
184 { __tpstrtab_##name, 0, reg, unreg, NULL }; \ 182 { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\
185 static struct tracepoint * const __tracepoint_ptr_##name __used \ 183 static struct tracepoint * const __tracepoint_ptr_##name __used \
186 __attribute__((section("__tracepoints_ptrs"))) = \ 184 __attribute__((section("__tracepoints_ptrs"))) = \
187 &__tracepoint_##name; 185 &__tracepoint_##name;
188 186
189#define DEFINE_TRACE(name) \ 187#define DEFINE_TRACE(name) \
diff --git a/kernel/Makefile b/kernel/Makefile
index 85cbfb31e73e..e9cf19155b46 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -21,7 +21,6 @@ CFLAGS_REMOVE_mutex-debug.o = -pg
21CFLAGS_REMOVE_rtmutex-debug.o = -pg 21CFLAGS_REMOVE_rtmutex-debug.o = -pg
22CFLAGS_REMOVE_cgroup-debug.o = -pg 22CFLAGS_REMOVE_cgroup-debug.o = -pg
23CFLAGS_REMOVE_sched_clock.o = -pg 23CFLAGS_REMOVE_sched_clock.o = -pg
24CFLAGS_REMOVE_perf_event.o = -pg
25CFLAGS_REMOVE_irq_work.o = -pg 24CFLAGS_REMOVE_irq_work.o = -pg
26endif 25endif
27 26
@@ -103,8 +102,9 @@ obj-$(CONFIG_RING_BUFFER) += trace/
103obj-$(CONFIG_TRACEPOINTS) += trace/ 102obj-$(CONFIG_TRACEPOINTS) += trace/
104obj-$(CONFIG_SMP) += sched_cpupri.o 103obj-$(CONFIG_SMP) += sched_cpupri.o
105obj-$(CONFIG_IRQ_WORK) += irq_work.o 104obj-$(CONFIG_IRQ_WORK) += irq_work.o
106obj-$(CONFIG_PERF_EVENTS) += perf_event.o 105
107obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o 106obj-$(CONFIG_PERF_EVENTS) += events/
107
108obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o 108obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
109obj-$(CONFIG_PADATA) += padata.o 109obj-$(CONFIG_PADATA) += padata.o
110obj-$(CONFIG_CRASH_DUMP) += crash_dump.o 110obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
diff --git a/kernel/events/Makefile b/kernel/events/Makefile
new file mode 100644
index 000000000000..1ce23d3d8394
--- /dev/null
+++ b/kernel/events/Makefile
@@ -0,0 +1,6 @@
1ifdef CONFIG_FUNCTION_TRACER
2CFLAGS_REMOVE_core.o = -pg
3endif
4
5obj-y := core.o
6obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
diff --git a/kernel/perf_event.c b/kernel/events/core.c
index 8e81a9860a0d..0fc34a370ba4 100644
--- a/kernel/perf_event.c
+++ b/kernel/events/core.c
@@ -2,8 +2,8 @@
2 * Performance events core code: 2 * Performance events core code:
3 * 3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8 * 8 *
9 * For licensing details see kernel-base/COPYING 9 * For licensing details see kernel-base/COPYING
@@ -39,10 +39,10 @@
39#include <asm/irq_regs.h> 39#include <asm/irq_regs.h>
40 40
41struct remote_function_call { 41struct remote_function_call {
42 struct task_struct *p; 42 struct task_struct *p;
43 int (*func)(void *info); 43 int (*func)(void *info);
44 void *info; 44 void *info;
45 int ret; 45 int ret;
46}; 46};
47 47
48static void remote_function(void *data) 48static void remote_function(void *data)
@@ -76,10 +76,10 @@ static int
76task_function_call(struct task_struct *p, int (*func) (void *info), void *info) 76task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
77{ 77{
78 struct remote_function_call data = { 78 struct remote_function_call data = {
79 .p = p, 79 .p = p,
80 .func = func, 80 .func = func,
81 .info = info, 81 .info = info,
82 .ret = -ESRCH, /* No such (running) process */ 82 .ret = -ESRCH, /* No such (running) process */
83 }; 83 };
84 84
85 if (task_curr(p)) 85 if (task_curr(p))
@@ -100,10 +100,10 @@ task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
100static int cpu_function_call(int cpu, int (*func) (void *info), void *info) 100static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
101{ 101{
102 struct remote_function_call data = { 102 struct remote_function_call data = {
103 .p = NULL, 103 .p = NULL,
104 .func = func, 104 .func = func,
105 .info = info, 105 .info = info,
106 .ret = -ENXIO, /* No such CPU */ 106 .ret = -ENXIO, /* No such CPU */
107 }; 107 };
108 108
109 smp_call_function_single(cpu, remote_function, &data, 1); 109 smp_call_function_single(cpu, remote_function, &data, 1);
@@ -125,7 +125,7 @@ enum event_type_t {
125 * perf_sched_events : >0 events exist 125 * perf_sched_events : >0 events exist
126 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu 126 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
127 */ 127 */
128atomic_t perf_sched_events __read_mostly; 128struct jump_label_key perf_sched_events __read_mostly;
129static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); 129static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
130 130
131static atomic_t nr_mmap_events __read_mostly; 131static atomic_t nr_mmap_events __read_mostly;
@@ -5429,7 +5429,7 @@ fail:
5429 return err; 5429 return err;
5430} 5430}
5431 5431
5432atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; 5432struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
5433 5433
5434static void sw_perf_event_destroy(struct perf_event *event) 5434static void sw_perf_event_destroy(struct perf_event *event)
5435{ 5435{
@@ -7445,11 +7445,11 @@ static void perf_cgroup_exit(struct cgroup_subsys *ss, struct cgroup *cgrp,
7445} 7445}
7446 7446
7447struct cgroup_subsys perf_subsys = { 7447struct cgroup_subsys perf_subsys = {
7448 .name = "perf_event", 7448 .name = "perf_event",
7449 .subsys_id = perf_subsys_id, 7449 .subsys_id = perf_subsys_id,
7450 .create = perf_cgroup_create, 7450 .create = perf_cgroup_create,
7451 .destroy = perf_cgroup_destroy, 7451 .destroy = perf_cgroup_destroy,
7452 .exit = perf_cgroup_exit, 7452 .exit = perf_cgroup_exit,
7453 .attach = perf_cgroup_attach, 7453 .attach = perf_cgroup_attach,
7454}; 7454};
7455#endif /* CONFIG_CGROUP_PERF */ 7455#endif /* CONFIG_CGROUP_PERF */
diff --git a/kernel/hw_breakpoint.c b/kernel/events/hw_breakpoint.c
index 086adf25a55e..086adf25a55e 100644
--- a/kernel/hw_breakpoint.c
+++ b/kernel/events/hw_breakpoint.c
diff --git a/kernel/extable.c b/kernel/extable.c
index 7f8f263f8524..c2d625fcda77 100644
--- a/kernel/extable.c
+++ b/kernel/extable.c
@@ -72,6 +72,14 @@ int core_kernel_text(unsigned long addr)
72 return 0; 72 return 0;
73} 73}
74 74
75int core_kernel_data(unsigned long addr)
76{
77 if (addr >= (unsigned long)_sdata &&
78 addr < (unsigned long)_edata)
79 return 1;
80 return 0;
81}
82
75int __kernel_text_address(unsigned long addr) 83int __kernel_text_address(unsigned long addr)
76{ 84{
77 if (core_kernel_text(addr)) 85 if (core_kernel_text(addr))
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 3b79bd938330..74d1c099fbd1 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -2,43 +2,23 @@
2 * jump label support 2 * jump label support
3 * 3 *
4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> 4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
5 * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com>
5 * 6 *
6 */ 7 */
7#include <linux/jump_label.h>
8#include <linux/memory.h> 8#include <linux/memory.h>
9#include <linux/uaccess.h> 9#include <linux/uaccess.h>
10#include <linux/module.h> 10#include <linux/module.h>
11#include <linux/list.h> 11#include <linux/list.h>
12#include <linux/jhash.h>
13#include <linux/slab.h> 12#include <linux/slab.h>
14#include <linux/sort.h> 13#include <linux/sort.h>
15#include <linux/err.h> 14#include <linux/err.h>
15#include <linux/jump_label.h>
16 16
17#ifdef HAVE_JUMP_LABEL 17#ifdef HAVE_JUMP_LABEL
18 18
19#define JUMP_LABEL_HASH_BITS 6
20#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
21static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
22
23/* mutex to protect coming/going of the the jump_label table */ 19/* mutex to protect coming/going of the the jump_label table */
24static DEFINE_MUTEX(jump_label_mutex); 20static DEFINE_MUTEX(jump_label_mutex);
25 21
26struct jump_label_entry {
27 struct hlist_node hlist;
28 struct jump_entry *table;
29 int nr_entries;
30 /* hang modules off here */
31 struct hlist_head modules;
32 unsigned long key;
33};
34
35struct jump_label_module_entry {
36 struct hlist_node hlist;
37 struct jump_entry *table;
38 int nr_entries;
39 struct module *mod;
40};
41
42void jump_label_lock(void) 22void jump_label_lock(void)
43{ 23{
44 mutex_lock(&jump_label_mutex); 24 mutex_lock(&jump_label_mutex);
@@ -49,6 +29,11 @@ void jump_label_unlock(void)
49 mutex_unlock(&jump_label_mutex); 29 mutex_unlock(&jump_label_mutex);
50} 30}
51 31
32bool jump_label_enabled(struct jump_label_key *key)
33{
34 return !!atomic_read(&key->enabled);
35}
36
52static int jump_label_cmp(const void *a, const void *b) 37static int jump_label_cmp(const void *a, const void *b)
53{ 38{
54 const struct jump_entry *jea = a; 39 const struct jump_entry *jea = a;
@@ -64,7 +49,7 @@ static int jump_label_cmp(const void *a, const void *b)
64} 49}
65 50
66static void 51static void
67sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop) 52jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
68{ 53{
69 unsigned long size; 54 unsigned long size;
70 55
@@ -73,118 +58,25 @@ sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
73 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); 58 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
74} 59}
75 60
76static struct jump_label_entry *get_jump_label_entry(jump_label_t key) 61static void jump_label_update(struct jump_label_key *key, int enable);
77{
78 struct hlist_head *head;
79 struct hlist_node *node;
80 struct jump_label_entry *e;
81 u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
82
83 head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
84 hlist_for_each_entry(e, node, head, hlist) {
85 if (key == e->key)
86 return e;
87 }
88 return NULL;
89}
90 62
91static struct jump_label_entry * 63void jump_label_inc(struct jump_label_key *key)
92add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
93{ 64{
94 struct hlist_head *head; 65 if (atomic_inc_not_zero(&key->enabled))
95 struct jump_label_entry *e; 66 return;
96 u32 hash;
97
98 e = get_jump_label_entry(key);
99 if (e)
100 return ERR_PTR(-EEXIST);
101
102 e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
103 if (!e)
104 return ERR_PTR(-ENOMEM);
105
106 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
107 head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
108 e->key = key;
109 e->table = table;
110 e->nr_entries = nr_entries;
111 INIT_HLIST_HEAD(&(e->modules));
112 hlist_add_head(&e->hlist, head);
113 return e;
114}
115 67
116static int 68 jump_label_lock();
117build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop) 69 if (atomic_add_return(1, &key->enabled) == 1)
118{ 70 jump_label_update(key, JUMP_LABEL_ENABLE);
119 struct jump_entry *iter, *iter_begin; 71 jump_label_unlock();
120 struct jump_label_entry *entry;
121 int count;
122
123 sort_jump_label_entries(start, stop);
124 iter = start;
125 while (iter < stop) {
126 entry = get_jump_label_entry(iter->key);
127 if (!entry) {
128 iter_begin = iter;
129 count = 0;
130 while ((iter < stop) &&
131 (iter->key == iter_begin->key)) {
132 iter++;
133 count++;
134 }
135 entry = add_jump_label_entry(iter_begin->key,
136 count, iter_begin);
137 if (IS_ERR(entry))
138 return PTR_ERR(entry);
139 } else {
140 WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
141 return -1;
142 }
143 }
144 return 0;
145} 72}
146 73
147/*** 74void jump_label_dec(struct jump_label_key *key)
148 * jump_label_update - update jump label text
149 * @key - key value associated with a a jump label
150 * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
151 *
152 * Will enable/disable the jump for jump label @key, depending on the
153 * value of @type.
154 *
155 */
156
157void jump_label_update(unsigned long key, enum jump_label_type type)
158{ 75{
159 struct jump_entry *iter; 76 if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex))
160 struct jump_label_entry *entry; 77 return;
161 struct hlist_node *module_node;
162 struct jump_label_module_entry *e_module;
163 int count;
164 78
165 jump_label_lock(); 79 jump_label_update(key, JUMP_LABEL_DISABLE);
166 entry = get_jump_label_entry((jump_label_t)key);
167 if (entry) {
168 count = entry->nr_entries;
169 iter = entry->table;
170 while (count--) {
171 if (kernel_text_address(iter->code))
172 arch_jump_label_transform(iter, type);
173 iter++;
174 }
175 /* eanble/disable jump labels in modules */
176 hlist_for_each_entry(e_module, module_node, &(entry->modules),
177 hlist) {
178 count = e_module->nr_entries;
179 iter = e_module->table;
180 while (count--) {
181 if (iter->key &&
182 kernel_text_address(iter->code))
183 arch_jump_label_transform(iter, type);
184 iter++;
185 }
186 }
187 }
188 jump_label_unlock(); 80 jump_label_unlock();
189} 81}
190 82
@@ -197,77 +89,33 @@ static int addr_conflict(struct jump_entry *entry, void *start, void *end)
197 return 0; 89 return 0;
198} 90}
199 91
200#ifdef CONFIG_MODULES 92static int __jump_label_text_reserved(struct jump_entry *iter_start,
201 93 struct jump_entry *iter_stop, void *start, void *end)
202static int module_conflict(void *start, void *end)
203{ 94{
204 struct hlist_head *head;
205 struct hlist_node *node, *node_next, *module_node, *module_node_next;
206 struct jump_label_entry *e;
207 struct jump_label_module_entry *e_module;
208 struct jump_entry *iter; 95 struct jump_entry *iter;
209 int i, count;
210 int conflict = 0;
211
212 for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
213 head = &jump_label_table[i];
214 hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
215 hlist_for_each_entry_safe(e_module, module_node,
216 module_node_next,
217 &(e->modules), hlist) {
218 count = e_module->nr_entries;
219 iter = e_module->table;
220 while (count--) {
221 if (addr_conflict(iter, start, end)) {
222 conflict = 1;
223 goto out;
224 }
225 iter++;
226 }
227 }
228 }
229 }
230out:
231 return conflict;
232}
233
234#endif
235
236/***
237 * jump_label_text_reserved - check if addr range is reserved
238 * @start: start text addr
239 * @end: end text addr
240 *
241 * checks if the text addr located between @start and @end
242 * overlaps with any of the jump label patch addresses. Code
243 * that wants to modify kernel text should first verify that
244 * it does not overlap with any of the jump label addresses.
245 * Caller must hold jump_label_mutex.
246 *
247 * returns 1 if there is an overlap, 0 otherwise
248 */
249int jump_label_text_reserved(void *start, void *end)
250{
251 struct jump_entry *iter;
252 struct jump_entry *iter_start = __start___jump_table;
253 struct jump_entry *iter_stop = __start___jump_table;
254 int conflict = 0;
255 96
256 iter = iter_start; 97 iter = iter_start;
257 while (iter < iter_stop) { 98 while (iter < iter_stop) {
258 if (addr_conflict(iter, start, end)) { 99 if (addr_conflict(iter, start, end))
259 conflict = 1; 100 return 1;
260 goto out;
261 }
262 iter++; 101 iter++;
263 } 102 }
264 103
265 /* now check modules */ 104 return 0;
266#ifdef CONFIG_MODULES 105}
267 conflict = module_conflict(start, end); 106
268#endif 107static void __jump_label_update(struct jump_label_key *key,
269out: 108 struct jump_entry *entry, int enable)
270 return conflict; 109{
110 for (; entry->key == (jump_label_t)(unsigned long)key; entry++) {
111 /*
112 * entry->code set to 0 invalidates module init text sections
113 * kernel_text_address() verifies we are not in core kernel
114 * init code, see jump_label_invalidate_module_init().
115 */
116 if (entry->code && kernel_text_address(entry->code))
117 arch_jump_label_transform(entry, enable);
118 }
271} 119}
272 120
273/* 121/*
@@ -277,142 +125,173 @@ void __weak arch_jump_label_text_poke_early(jump_label_t addr)
277{ 125{
278} 126}
279 127
280static __init int init_jump_label(void) 128static __init int jump_label_init(void)
281{ 129{
282 int ret;
283 struct jump_entry *iter_start = __start___jump_table; 130 struct jump_entry *iter_start = __start___jump_table;
284 struct jump_entry *iter_stop = __stop___jump_table; 131 struct jump_entry *iter_stop = __stop___jump_table;
132 struct jump_label_key *key = NULL;
285 struct jump_entry *iter; 133 struct jump_entry *iter;
286 134
287 jump_label_lock(); 135 jump_label_lock();
288 ret = build_jump_label_hashtable(__start___jump_table, 136 jump_label_sort_entries(iter_start, iter_stop);
289 __stop___jump_table); 137
290 iter = iter_start; 138 for (iter = iter_start; iter < iter_stop; iter++) {
291 while (iter < iter_stop) {
292 arch_jump_label_text_poke_early(iter->code); 139 arch_jump_label_text_poke_early(iter->code);
293 iter++; 140 if (iter->key == (jump_label_t)(unsigned long)key)
141 continue;
142
143 key = (struct jump_label_key *)(unsigned long)iter->key;
144 atomic_set(&key->enabled, 0);
145 key->entries = iter;
146#ifdef CONFIG_MODULES
147 key->next = NULL;
148#endif
294 } 149 }
295 jump_label_unlock(); 150 jump_label_unlock();
296 return ret; 151
152 return 0;
297} 153}
298early_initcall(init_jump_label); 154early_initcall(jump_label_init);
299 155
300#ifdef CONFIG_MODULES 156#ifdef CONFIG_MODULES
301 157
302static struct jump_label_module_entry * 158struct jump_label_mod {
303add_jump_label_module_entry(struct jump_label_entry *entry, 159 struct jump_label_mod *next;
304 struct jump_entry *iter_begin, 160 struct jump_entry *entries;
305 int count, struct module *mod) 161 struct module *mod;
162};
163
164static int __jump_label_mod_text_reserved(void *start, void *end)
165{
166 struct module *mod;
167
168 mod = __module_text_address((unsigned long)start);
169 if (!mod)
170 return 0;
171
172 WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
173
174 return __jump_label_text_reserved(mod->jump_entries,
175 mod->jump_entries + mod->num_jump_entries,
176 start, end);
177}
178
179static void __jump_label_mod_update(struct jump_label_key *key, int enable)
180{
181 struct jump_label_mod *mod = key->next;
182
183 while (mod) {
184 __jump_label_update(key, mod->entries, enable);
185 mod = mod->next;
186 }
187}
188
189/***
190 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
191 * @mod: module to patch
192 *
193 * Allow for run-time selection of the optimal nops. Before the module
194 * loads patch these with arch_get_jump_label_nop(), which is specified by
195 * the arch specific jump label code.
196 */
197void jump_label_apply_nops(struct module *mod)
306{ 198{
307 struct jump_label_module_entry *e; 199 struct jump_entry *iter_start = mod->jump_entries;
308 200 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
309 e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL); 201 struct jump_entry *iter;
310 if (!e) 202
311 return ERR_PTR(-ENOMEM); 203 /* if the module doesn't have jump label entries, just return */
312 e->mod = mod; 204 if (iter_start == iter_stop)
313 e->nr_entries = count; 205 return;
314 e->table = iter_begin; 206
315 hlist_add_head(&e->hlist, &entry->modules); 207 for (iter = iter_start; iter < iter_stop; iter++)
316 return e; 208 arch_jump_label_text_poke_early(iter->code);
317} 209}
318 210
319static int add_jump_label_module(struct module *mod) 211static int jump_label_add_module(struct module *mod)
320{ 212{
321 struct jump_entry *iter, *iter_begin; 213 struct jump_entry *iter_start = mod->jump_entries;
322 struct jump_label_entry *entry; 214 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
323 struct jump_label_module_entry *module_entry; 215 struct jump_entry *iter;
324 int count; 216 struct jump_label_key *key = NULL;
217 struct jump_label_mod *jlm;
325 218
326 /* if the module doesn't have jump label entries, just return */ 219 /* if the module doesn't have jump label entries, just return */
327 if (!mod->num_jump_entries) 220 if (iter_start == iter_stop)
328 return 0; 221 return 0;
329 222
330 sort_jump_label_entries(mod->jump_entries, 223 jump_label_sort_entries(iter_start, iter_stop);
331 mod->jump_entries + mod->num_jump_entries); 224
332 iter = mod->jump_entries; 225 for (iter = iter_start; iter < iter_stop; iter++) {
333 while (iter < mod->jump_entries + mod->num_jump_entries) { 226 if (iter->key == (jump_label_t)(unsigned long)key)
334 entry = get_jump_label_entry(iter->key); 227 continue;
335 iter_begin = iter; 228
336 count = 0; 229 key = (struct jump_label_key *)(unsigned long)iter->key;
337 while ((iter < mod->jump_entries + mod->num_jump_entries) && 230
338 (iter->key == iter_begin->key)) { 231 if (__module_address(iter->key) == mod) {
339 iter++; 232 atomic_set(&key->enabled, 0);
340 count++; 233 key->entries = iter;
341 } 234 key->next = NULL;
342 if (!entry) { 235 continue;
343 entry = add_jump_label_entry(iter_begin->key, 0, NULL);
344 if (IS_ERR(entry))
345 return PTR_ERR(entry);
346 } 236 }
347 module_entry = add_jump_label_module_entry(entry, iter_begin, 237
348 count, mod); 238 jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
349 if (IS_ERR(module_entry)) 239 if (!jlm)
350 return PTR_ERR(module_entry); 240 return -ENOMEM;
241
242 jlm->mod = mod;
243 jlm->entries = iter;
244 jlm->next = key->next;
245 key->next = jlm;
246
247 if (jump_label_enabled(key))
248 __jump_label_update(key, iter, JUMP_LABEL_ENABLE);
351 } 249 }
250
352 return 0; 251 return 0;
353} 252}
354 253
355static void remove_jump_label_module(struct module *mod) 254static void jump_label_del_module(struct module *mod)
356{ 255{
357 struct hlist_head *head; 256 struct jump_entry *iter_start = mod->jump_entries;
358 struct hlist_node *node, *node_next, *module_node, *module_node_next; 257 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
359 struct jump_label_entry *e; 258 struct jump_entry *iter;
360 struct jump_label_module_entry *e_module; 259 struct jump_label_key *key = NULL;
361 int i; 260 struct jump_label_mod *jlm, **prev;
362 261
363 /* if the module doesn't have jump label entries, just return */ 262 for (iter = iter_start; iter < iter_stop; iter++) {
364 if (!mod->num_jump_entries) 263 if (iter->key == (jump_label_t)(unsigned long)key)
365 return; 264 continue;
265
266 key = (struct jump_label_key *)(unsigned long)iter->key;
267
268 if (__module_address(iter->key) == mod)
269 continue;
270
271 prev = &key->next;
272 jlm = key->next;
366 273
367 for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { 274 while (jlm && jlm->mod != mod) {
368 head = &jump_label_table[i]; 275 prev = &jlm->next;
369 hlist_for_each_entry_safe(e, node, node_next, head, hlist) { 276 jlm = jlm->next;
370 hlist_for_each_entry_safe(e_module, module_node, 277 }
371 module_node_next, 278
372 &(e->modules), hlist) { 279 if (jlm) {
373 if (e_module->mod == mod) { 280 *prev = jlm->next;
374 hlist_del(&e_module->hlist); 281 kfree(jlm);
375 kfree(e_module);
376 }
377 }
378 if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
379 hlist_del(&e->hlist);
380 kfree(e);
381 }
382 } 282 }
383 } 283 }
384} 284}
385 285
386static void remove_jump_label_module_init(struct module *mod) 286static void jump_label_invalidate_module_init(struct module *mod)
387{ 287{
388 struct hlist_head *head; 288 struct jump_entry *iter_start = mod->jump_entries;
389 struct hlist_node *node, *node_next, *module_node, *module_node_next; 289 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
390 struct jump_label_entry *e;
391 struct jump_label_module_entry *e_module;
392 struct jump_entry *iter; 290 struct jump_entry *iter;
393 int i, count;
394
395 /* if the module doesn't have jump label entries, just return */
396 if (!mod->num_jump_entries)
397 return;
398 291
399 for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { 292 for (iter = iter_start; iter < iter_stop; iter++) {
400 head = &jump_label_table[i]; 293 if (within_module_init(iter->code, mod))
401 hlist_for_each_entry_safe(e, node, node_next, head, hlist) { 294 iter->code = 0;
402 hlist_for_each_entry_safe(e_module, module_node,
403 module_node_next,
404 &(e->modules), hlist) {
405 if (e_module->mod != mod)
406 continue;
407 count = e_module->nr_entries;
408 iter = e_module->table;
409 while (count--) {
410 if (within_module_init(iter->code, mod))
411 iter->key = 0;
412 iter++;
413 }
414 }
415 }
416 } 295 }
417} 296}
418 297
@@ -426,59 +305,77 @@ jump_label_module_notify(struct notifier_block *self, unsigned long val,
426 switch (val) { 305 switch (val) {
427 case MODULE_STATE_COMING: 306 case MODULE_STATE_COMING:
428 jump_label_lock(); 307 jump_label_lock();
429 ret = add_jump_label_module(mod); 308 ret = jump_label_add_module(mod);
430 if (ret) 309 if (ret)
431 remove_jump_label_module(mod); 310 jump_label_del_module(mod);
432 jump_label_unlock(); 311 jump_label_unlock();
433 break; 312 break;
434 case MODULE_STATE_GOING: 313 case MODULE_STATE_GOING:
435 jump_label_lock(); 314 jump_label_lock();
436 remove_jump_label_module(mod); 315 jump_label_del_module(mod);
437 jump_label_unlock(); 316 jump_label_unlock();
438 break; 317 break;
439 case MODULE_STATE_LIVE: 318 case MODULE_STATE_LIVE:
440 jump_label_lock(); 319 jump_label_lock();
441 remove_jump_label_module_init(mod); 320 jump_label_invalidate_module_init(mod);
442 jump_label_unlock(); 321 jump_label_unlock();
443 break; 322 break;
444 } 323 }
445 return ret;
446}
447 324
448/*** 325 return notifier_from_errno(ret);
449 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
450 * @mod: module to patch
451 *
452 * Allow for run-time selection of the optimal nops. Before the module
453 * loads patch these with arch_get_jump_label_nop(), which is specified by
454 * the arch specific jump label code.
455 */
456void jump_label_apply_nops(struct module *mod)
457{
458 struct jump_entry *iter;
459
460 /* if the module doesn't have jump label entries, just return */
461 if (!mod->num_jump_entries)
462 return;
463
464 iter = mod->jump_entries;
465 while (iter < mod->jump_entries + mod->num_jump_entries) {
466 arch_jump_label_text_poke_early(iter->code);
467 iter++;
468 }
469} 326}
470 327
471struct notifier_block jump_label_module_nb = { 328struct notifier_block jump_label_module_nb = {
472 .notifier_call = jump_label_module_notify, 329 .notifier_call = jump_label_module_notify,
473 .priority = 0, 330 .priority = 1, /* higher than tracepoints */
474}; 331};
475 332
476static __init int init_jump_label_module(void) 333static __init int jump_label_init_module(void)
477{ 334{
478 return register_module_notifier(&jump_label_module_nb); 335 return register_module_notifier(&jump_label_module_nb);
479} 336}
480early_initcall(init_jump_label_module); 337early_initcall(jump_label_init_module);
481 338
482#endif /* CONFIG_MODULES */ 339#endif /* CONFIG_MODULES */
483 340
341/***
342 * jump_label_text_reserved - check if addr range is reserved
343 * @start: start text addr
344 * @end: end text addr
345 *
346 * checks if the text addr located between @start and @end
347 * overlaps with any of the jump label patch addresses. Code
348 * that wants to modify kernel text should first verify that
349 * it does not overlap with any of the jump label addresses.
350 * Caller must hold jump_label_mutex.
351 *
352 * returns 1 if there is an overlap, 0 otherwise
353 */
354int jump_label_text_reserved(void *start, void *end)
355{
356 int ret = __jump_label_text_reserved(__start___jump_table,
357 __stop___jump_table, start, end);
358
359 if (ret)
360 return ret;
361
362#ifdef CONFIG_MODULES
363 ret = __jump_label_mod_text_reserved(start, end);
364#endif
365 return ret;
366}
367
368static void jump_label_update(struct jump_label_key *key, int enable)
369{
370 struct jump_entry *entry = key->entries;
371
372 /* if there are no users, entry can be NULL */
373 if (entry)
374 __jump_label_update(key, entry, enable);
375
376#ifdef CONFIG_MODULES
377 __jump_label_mod_update(key, enable);
378#endif
379}
380
484#endif 381#endif
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index ee24fa1935ac..d017c2c82c44 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -39,20 +39,26 @@
39#include "trace_stat.h" 39#include "trace_stat.h"
40 40
41#define FTRACE_WARN_ON(cond) \ 41#define FTRACE_WARN_ON(cond) \
42 do { \ 42 ({ \
43 if (WARN_ON(cond)) \ 43 int ___r = cond; \
44 if (WARN_ON(___r)) \
44 ftrace_kill(); \ 45 ftrace_kill(); \
45 } while (0) 46 ___r; \
47 })
46 48
47#define FTRACE_WARN_ON_ONCE(cond) \ 49#define FTRACE_WARN_ON_ONCE(cond) \
48 do { \ 50 ({ \
49 if (WARN_ON_ONCE(cond)) \ 51 int ___r = cond; \
52 if (WARN_ON_ONCE(___r)) \
50 ftrace_kill(); \ 53 ftrace_kill(); \
51 } while (0) 54 ___r; \
55 })
52 56
53/* hash bits for specific function selection */ 57/* hash bits for specific function selection */
54#define FTRACE_HASH_BITS 7 58#define FTRACE_HASH_BITS 7
55#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) 59#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
60#define FTRACE_HASH_DEFAULT_BITS 10
61#define FTRACE_HASH_MAX_BITS 12
56 62
57/* ftrace_enabled is a method to turn ftrace on or off */ 63/* ftrace_enabled is a method to turn ftrace on or off */
58int ftrace_enabled __read_mostly; 64int ftrace_enabled __read_mostly;
@@ -81,23 +87,29 @@ static struct ftrace_ops ftrace_list_end __read_mostly =
81 .func = ftrace_stub, 87 .func = ftrace_stub,
82}; 88};
83 89
84static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end; 90static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
91static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
85ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 92ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
86ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; 93ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
87ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; 94ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
95static struct ftrace_ops global_ops;
96
97static void
98ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
88 99
89/* 100/*
90 * Traverse the ftrace_list, invoking all entries. The reason that we 101 * Traverse the ftrace_global_list, invoking all entries. The reason that we
91 * can use rcu_dereference_raw() is that elements removed from this list 102 * can use rcu_dereference_raw() is that elements removed from this list
92 * are simply leaked, so there is no need to interact with a grace-period 103 * are simply leaked, so there is no need to interact with a grace-period
93 * mechanism. The rcu_dereference_raw() calls are needed to handle 104 * mechanism. The rcu_dereference_raw() calls are needed to handle
94 * concurrent insertions into the ftrace_list. 105 * concurrent insertions into the ftrace_global_list.
95 * 106 *
96 * Silly Alpha and silly pointer-speculation compiler optimizations! 107 * Silly Alpha and silly pointer-speculation compiler optimizations!
97 */ 108 */
98static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) 109static void ftrace_global_list_func(unsigned long ip,
110 unsigned long parent_ip)
99{ 111{
100 struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/ 112 struct ftrace_ops *op = rcu_dereference_raw(ftrace_global_list); /*see above*/
101 113
102 while (op != &ftrace_list_end) { 114 while (op != &ftrace_list_end) {
103 op->func(ip, parent_ip); 115 op->func(ip, parent_ip);
@@ -147,46 +159,69 @@ static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
147} 159}
148#endif 160#endif
149 161
150static int __register_ftrace_function(struct ftrace_ops *ops) 162static void update_global_ops(void)
151{ 163{
152 ops->next = ftrace_list; 164 ftrace_func_t func;
165
153 /* 166 /*
154 * We are entering ops into the ftrace_list but another 167 * If there's only one function registered, then call that
155 * CPU might be walking that list. We need to make sure 168 * function directly. Otherwise, we need to iterate over the
156 * the ops->next pointer is valid before another CPU sees 169 * registered callers.
157 * the ops pointer included into the ftrace_list.
158 */ 170 */
159 rcu_assign_pointer(ftrace_list, ops); 171 if (ftrace_global_list == &ftrace_list_end ||
172 ftrace_global_list->next == &ftrace_list_end)
173 func = ftrace_global_list->func;
174 else
175 func = ftrace_global_list_func;
160 176
161 if (ftrace_enabled) { 177 /* If we filter on pids, update to use the pid function */
162 ftrace_func_t func; 178 if (!list_empty(&ftrace_pids)) {
179 set_ftrace_pid_function(func);
180 func = ftrace_pid_func;
181 }
163 182
164 if (ops->next == &ftrace_list_end) 183 global_ops.func = func;
165 func = ops->func; 184}
166 else
167 func = ftrace_list_func;
168 185
169 if (!list_empty(&ftrace_pids)) { 186static void update_ftrace_function(void)
170 set_ftrace_pid_function(func); 187{
171 func = ftrace_pid_func; 188 ftrace_func_t func;
172 } 189
190 update_global_ops();
191
192 /*
193 * If we are at the end of the list and this ops is
194 * not dynamic, then have the mcount trampoline call
195 * the function directly
196 */
197 if (ftrace_ops_list == &ftrace_list_end ||
198 (ftrace_ops_list->next == &ftrace_list_end &&
199 !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC)))
200 func = ftrace_ops_list->func;
201 else
202 func = ftrace_ops_list_func;
173 203
174 /*
175 * For one func, simply call it directly.
176 * For more than one func, call the chain.
177 */
178#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 204#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
179 ftrace_trace_function = func; 205 ftrace_trace_function = func;
180#else 206#else
181 __ftrace_trace_function = func; 207 __ftrace_trace_function = func;
182 ftrace_trace_function = ftrace_test_stop_func; 208 ftrace_trace_function = ftrace_test_stop_func;
183#endif 209#endif
184 } 210}
185 211
186 return 0; 212static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
213{
214 ops->next = *list;
215 /*
216 * We are entering ops into the list but another
217 * CPU might be walking that list. We need to make sure
218 * the ops->next pointer is valid before another CPU sees
219 * the ops pointer included into the list.
220 */
221 rcu_assign_pointer(*list, ops);
187} 222}
188 223
189static int __unregister_ftrace_function(struct ftrace_ops *ops) 224static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
190{ 225{
191 struct ftrace_ops **p; 226 struct ftrace_ops **p;
192 227
@@ -194,13 +229,12 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
194 * If we are removing the last function, then simply point 229 * If we are removing the last function, then simply point
195 * to the ftrace_stub. 230 * to the ftrace_stub.
196 */ 231 */
197 if (ftrace_list == ops && ops->next == &ftrace_list_end) { 232 if (*list == ops && ops->next == &ftrace_list_end) {
198 ftrace_trace_function = ftrace_stub; 233 *list = &ftrace_list_end;
199 ftrace_list = &ftrace_list_end;
200 return 0; 234 return 0;
201 } 235 }
202 236
203 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next) 237 for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
204 if (*p == ops) 238 if (*p == ops)
205 break; 239 break;
206 240
@@ -208,53 +242,83 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
208 return -1; 242 return -1;
209 243
210 *p = (*p)->next; 244 *p = (*p)->next;
245 return 0;
246}
211 247
212 if (ftrace_enabled) { 248static int __register_ftrace_function(struct ftrace_ops *ops)
213 /* If we only have one func left, then call that directly */ 249{
214 if (ftrace_list->next == &ftrace_list_end) { 250 if (ftrace_disabled)
215 ftrace_func_t func = ftrace_list->func; 251 return -ENODEV;
216 252
217 if (!list_empty(&ftrace_pids)) { 253 if (FTRACE_WARN_ON(ops == &global_ops))
218 set_ftrace_pid_function(func); 254 return -EINVAL;
219 func = ftrace_pid_func; 255
220 } 256 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
221#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 257 return -EBUSY;
222 ftrace_trace_function = func; 258
223#else 259 if (!core_kernel_data((unsigned long)ops))
224 __ftrace_trace_function = func; 260 ops->flags |= FTRACE_OPS_FL_DYNAMIC;
225#endif 261
226 } 262 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
227 } 263 int first = ftrace_global_list == &ftrace_list_end;
264 add_ftrace_ops(&ftrace_global_list, ops);
265 ops->flags |= FTRACE_OPS_FL_ENABLED;
266 if (first)
267 add_ftrace_ops(&ftrace_ops_list, &global_ops);
268 } else
269 add_ftrace_ops(&ftrace_ops_list, ops);
270
271 if (ftrace_enabled)
272 update_ftrace_function();
228 273
229 return 0; 274 return 0;
230} 275}
231 276
232static void ftrace_update_pid_func(void) 277static int __unregister_ftrace_function(struct ftrace_ops *ops)
233{ 278{
234 ftrace_func_t func; 279 int ret;
235 280
236 if (ftrace_trace_function == ftrace_stub) 281 if (ftrace_disabled)
237 return; 282 return -ENODEV;
238 283
239#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 284 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
240 func = ftrace_trace_function; 285 return -EBUSY;
241#else
242 func = __ftrace_trace_function;
243#endif
244 286
245 if (!list_empty(&ftrace_pids)) { 287 if (FTRACE_WARN_ON(ops == &global_ops))
246 set_ftrace_pid_function(func); 288 return -EINVAL;
247 func = ftrace_pid_func;
248 } else {
249 if (func == ftrace_pid_func)
250 func = ftrace_pid_function;
251 }
252 289
253#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 290 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
254 ftrace_trace_function = func; 291 ret = remove_ftrace_ops(&ftrace_global_list, ops);
255#else 292 if (!ret && ftrace_global_list == &ftrace_list_end)
256 __ftrace_trace_function = func; 293 ret = remove_ftrace_ops(&ftrace_ops_list, &global_ops);
257#endif 294 if (!ret)
295 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
296 } else
297 ret = remove_ftrace_ops(&ftrace_ops_list, ops);
298
299 if (ret < 0)
300 return ret;
301
302 if (ftrace_enabled)
303 update_ftrace_function();
304
305 /*
306 * Dynamic ops may be freed, we must make sure that all
307 * callers are done before leaving this function.
308 */
309 if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
310 synchronize_sched();
311
312 return 0;
313}
314
315static void ftrace_update_pid_func(void)
316{
317 /* Only do something if we are tracing something */
318 if (ftrace_trace_function == ftrace_stub)
319 return;
320
321 update_ftrace_function();
258} 322}
259 323
260#ifdef CONFIG_FUNCTION_PROFILER 324#ifdef CONFIG_FUNCTION_PROFILER
@@ -888,8 +952,35 @@ enum {
888 FTRACE_START_FUNC_RET = (1 << 3), 952 FTRACE_START_FUNC_RET = (1 << 3),
889 FTRACE_STOP_FUNC_RET = (1 << 4), 953 FTRACE_STOP_FUNC_RET = (1 << 4),
890}; 954};
955struct ftrace_func_entry {
956 struct hlist_node hlist;
957 unsigned long ip;
958};
891 959
892static int ftrace_filtered; 960struct ftrace_hash {
961 unsigned long size_bits;
962 struct hlist_head *buckets;
963 unsigned long count;
964 struct rcu_head rcu;
965};
966
967/*
968 * We make these constant because no one should touch them,
969 * but they are used as the default "empty hash", to avoid allocating
970 * it all the time. These are in a read only section such that if
971 * anyone does try to modify it, it will cause an exception.
972 */
973static const struct hlist_head empty_buckets[1];
974static const struct ftrace_hash empty_hash = {
975 .buckets = (struct hlist_head *)empty_buckets,
976};
977#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
978
979static struct ftrace_ops global_ops = {
980 .func = ftrace_stub,
981 .notrace_hash = EMPTY_HASH,
982 .filter_hash = EMPTY_HASH,
983};
893 984
894static struct dyn_ftrace *ftrace_new_addrs; 985static struct dyn_ftrace *ftrace_new_addrs;
895 986
@@ -912,6 +1003,269 @@ static struct ftrace_page *ftrace_pages;
912 1003
913static struct dyn_ftrace *ftrace_free_records; 1004static struct dyn_ftrace *ftrace_free_records;
914 1005
1006static struct ftrace_func_entry *
1007ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1008{
1009 unsigned long key;
1010 struct ftrace_func_entry *entry;
1011 struct hlist_head *hhd;
1012 struct hlist_node *n;
1013
1014 if (!hash->count)
1015 return NULL;
1016
1017 if (hash->size_bits > 0)
1018 key = hash_long(ip, hash->size_bits);
1019 else
1020 key = 0;
1021
1022 hhd = &hash->buckets[key];
1023
1024 hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
1025 if (entry->ip == ip)
1026 return entry;
1027 }
1028 return NULL;
1029}
1030
1031static void __add_hash_entry(struct ftrace_hash *hash,
1032 struct ftrace_func_entry *entry)
1033{
1034 struct hlist_head *hhd;
1035 unsigned long key;
1036
1037 if (hash->size_bits)
1038 key = hash_long(entry->ip, hash->size_bits);
1039 else
1040 key = 0;
1041
1042 hhd = &hash->buckets[key];
1043 hlist_add_head(&entry->hlist, hhd);
1044 hash->count++;
1045}
1046
1047static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1048{
1049 struct ftrace_func_entry *entry;
1050
1051 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1052 if (!entry)
1053 return -ENOMEM;
1054
1055 entry->ip = ip;
1056 __add_hash_entry(hash, entry);
1057
1058 return 0;
1059}
1060
1061static void
1062free_hash_entry(struct ftrace_hash *hash,
1063 struct ftrace_func_entry *entry)
1064{
1065 hlist_del(&entry->hlist);
1066 kfree(entry);
1067 hash->count--;
1068}
1069
1070static void
1071remove_hash_entry(struct ftrace_hash *hash,
1072 struct ftrace_func_entry *entry)
1073{
1074 hlist_del(&entry->hlist);
1075 hash->count--;
1076}
1077
1078static void ftrace_hash_clear(struct ftrace_hash *hash)
1079{
1080 struct hlist_head *hhd;
1081 struct hlist_node *tp, *tn;
1082 struct ftrace_func_entry *entry;
1083 int size = 1 << hash->size_bits;
1084 int i;
1085
1086 if (!hash->count)
1087 return;
1088
1089 for (i = 0; i < size; i++) {
1090 hhd = &hash->buckets[i];
1091 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
1092 free_hash_entry(hash, entry);
1093 }
1094 FTRACE_WARN_ON(hash->count);
1095}
1096
1097static void free_ftrace_hash(struct ftrace_hash *hash)
1098{
1099 if (!hash || hash == EMPTY_HASH)
1100 return;
1101 ftrace_hash_clear(hash);
1102 kfree(hash->buckets);
1103 kfree(hash);
1104}
1105
1106static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1107{
1108 struct ftrace_hash *hash;
1109
1110 hash = container_of(rcu, struct ftrace_hash, rcu);
1111 free_ftrace_hash(hash);
1112}
1113
1114static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1115{
1116 if (!hash || hash == EMPTY_HASH)
1117 return;
1118 call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1119}
1120
1121static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1122{
1123 struct ftrace_hash *hash;
1124 int size;
1125
1126 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1127 if (!hash)
1128 return NULL;
1129
1130 size = 1 << size_bits;
1131 hash->buckets = kzalloc(sizeof(*hash->buckets) * size, GFP_KERNEL);
1132
1133 if (!hash->buckets) {
1134 kfree(hash);
1135 return NULL;
1136 }
1137
1138 hash->size_bits = size_bits;
1139
1140 return hash;
1141}
1142
1143static struct ftrace_hash *
1144alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1145{
1146 struct ftrace_func_entry *entry;
1147 struct ftrace_hash *new_hash;
1148 struct hlist_node *tp;
1149 int size;
1150 int ret;
1151 int i;
1152
1153 new_hash = alloc_ftrace_hash(size_bits);
1154 if (!new_hash)
1155 return NULL;
1156
1157 /* Empty hash? */
1158 if (!hash || !hash->count)
1159 return new_hash;
1160
1161 size = 1 << hash->size_bits;
1162 for (i = 0; i < size; i++) {
1163 hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
1164 ret = add_hash_entry(new_hash, entry->ip);
1165 if (ret < 0)
1166 goto free_hash;
1167 }
1168 }
1169
1170 FTRACE_WARN_ON(new_hash->count != hash->count);
1171
1172 return new_hash;
1173
1174 free_hash:
1175 free_ftrace_hash(new_hash);
1176 return NULL;
1177}
1178
1179static int
1180ftrace_hash_move(struct ftrace_hash **dst, struct ftrace_hash *src)
1181{
1182 struct ftrace_func_entry *entry;
1183 struct hlist_node *tp, *tn;
1184 struct hlist_head *hhd;
1185 struct ftrace_hash *old_hash;
1186 struct ftrace_hash *new_hash;
1187 unsigned long key;
1188 int size = src->count;
1189 int bits = 0;
1190 int i;
1191
1192 /*
1193 * If the new source is empty, just free dst and assign it
1194 * the empty_hash.
1195 */
1196 if (!src->count) {
1197 free_ftrace_hash_rcu(*dst);
1198 rcu_assign_pointer(*dst, EMPTY_HASH);
1199 return 0;
1200 }
1201
1202 /*
1203 * Make the hash size about 1/2 the # found
1204 */
1205 for (size /= 2; size; size >>= 1)
1206 bits++;
1207
1208 /* Don't allocate too much */
1209 if (bits > FTRACE_HASH_MAX_BITS)
1210 bits = FTRACE_HASH_MAX_BITS;
1211
1212 new_hash = alloc_ftrace_hash(bits);
1213 if (!new_hash)
1214 return -ENOMEM;
1215
1216 size = 1 << src->size_bits;
1217 for (i = 0; i < size; i++) {
1218 hhd = &src->buckets[i];
1219 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
1220 if (bits > 0)
1221 key = hash_long(entry->ip, bits);
1222 else
1223 key = 0;
1224 remove_hash_entry(src, entry);
1225 __add_hash_entry(new_hash, entry);
1226 }
1227 }
1228
1229 old_hash = *dst;
1230 rcu_assign_pointer(*dst, new_hash);
1231 free_ftrace_hash_rcu(old_hash);
1232
1233 return 0;
1234}
1235
1236/*
1237 * Test the hashes for this ops to see if we want to call
1238 * the ops->func or not.
1239 *
1240 * It's a match if the ip is in the ops->filter_hash or
1241 * the filter_hash does not exist or is empty,
1242 * AND
1243 * the ip is not in the ops->notrace_hash.
1244 *
1245 * This needs to be called with preemption disabled as
1246 * the hashes are freed with call_rcu_sched().
1247 */
1248static int
1249ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
1250{
1251 struct ftrace_hash *filter_hash;
1252 struct ftrace_hash *notrace_hash;
1253 int ret;
1254
1255 filter_hash = rcu_dereference_raw(ops->filter_hash);
1256 notrace_hash = rcu_dereference_raw(ops->notrace_hash);
1257
1258 if ((!filter_hash || !filter_hash->count ||
1259 ftrace_lookup_ip(filter_hash, ip)) &&
1260 (!notrace_hash || !notrace_hash->count ||
1261 !ftrace_lookup_ip(notrace_hash, ip)))
1262 ret = 1;
1263 else
1264 ret = 0;
1265
1266 return ret;
1267}
1268
915/* 1269/*
916 * This is a double for. Do not use 'break' to break out of the loop, 1270 * This is a double for. Do not use 'break' to break out of the loop,
917 * you must use a goto. 1271 * you must use a goto.
@@ -926,6 +1280,105 @@ static struct dyn_ftrace *ftrace_free_records;
926 } \ 1280 } \
927 } 1281 }
928 1282
1283static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1284 int filter_hash,
1285 bool inc)
1286{
1287 struct ftrace_hash *hash;
1288 struct ftrace_hash *other_hash;
1289 struct ftrace_page *pg;
1290 struct dyn_ftrace *rec;
1291 int count = 0;
1292 int all = 0;
1293
1294 /* Only update if the ops has been registered */
1295 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1296 return;
1297
1298 /*
1299 * In the filter_hash case:
1300 * If the count is zero, we update all records.
1301 * Otherwise we just update the items in the hash.
1302 *
1303 * In the notrace_hash case:
1304 * We enable the update in the hash.
1305 * As disabling notrace means enabling the tracing,
1306 * and enabling notrace means disabling, the inc variable
1307 * gets inversed.
1308 */
1309 if (filter_hash) {
1310 hash = ops->filter_hash;
1311 other_hash = ops->notrace_hash;
1312 if (!hash || !hash->count)
1313 all = 1;
1314 } else {
1315 inc = !inc;
1316 hash = ops->notrace_hash;
1317 other_hash = ops->filter_hash;
1318 /*
1319 * If the notrace hash has no items,
1320 * then there's nothing to do.
1321 */
1322 if (hash && !hash->count)
1323 return;
1324 }
1325
1326 do_for_each_ftrace_rec(pg, rec) {
1327 int in_other_hash = 0;
1328 int in_hash = 0;
1329 int match = 0;
1330
1331 if (all) {
1332 /*
1333 * Only the filter_hash affects all records.
1334 * Update if the record is not in the notrace hash.
1335 */
1336 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1337 match = 1;
1338 } else {
1339 in_hash = hash && !!ftrace_lookup_ip(hash, rec->ip);
1340 in_other_hash = other_hash && !!ftrace_lookup_ip(other_hash, rec->ip);
1341
1342 /*
1343 *
1344 */
1345 if (filter_hash && in_hash && !in_other_hash)
1346 match = 1;
1347 else if (!filter_hash && in_hash &&
1348 (in_other_hash || !other_hash->count))
1349 match = 1;
1350 }
1351 if (!match)
1352 continue;
1353
1354 if (inc) {
1355 rec->flags++;
1356 if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1357 return;
1358 } else {
1359 if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1360 return;
1361 rec->flags--;
1362 }
1363 count++;
1364 /* Shortcut, if we handled all records, we are done. */
1365 if (!all && count == hash->count)
1366 return;
1367 } while_for_each_ftrace_rec();
1368}
1369
1370static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1371 int filter_hash)
1372{
1373 __ftrace_hash_rec_update(ops, filter_hash, 0);
1374}
1375
1376static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1377 int filter_hash)
1378{
1379 __ftrace_hash_rec_update(ops, filter_hash, 1);
1380}
1381
929static void ftrace_free_rec(struct dyn_ftrace *rec) 1382static void ftrace_free_rec(struct dyn_ftrace *rec)
930{ 1383{
931 rec->freelist = ftrace_free_records; 1384 rec->freelist = ftrace_free_records;
@@ -1047,18 +1500,18 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1047 ftrace_addr = (unsigned long)FTRACE_ADDR; 1500 ftrace_addr = (unsigned long)FTRACE_ADDR;
1048 1501
1049 /* 1502 /*
1050 * If this record is not to be traced or we want to disable it, 1503 * If we are enabling tracing:
1051 * then disable it. 1504 *
1505 * If the record has a ref count, then we need to enable it
1506 * because someone is using it.
1052 * 1507 *
1053 * If we want to enable it and filtering is off, then enable it. 1508 * Otherwise we make sure its disabled.
1054 * 1509 *
1055 * If we want to enable it and filtering is on, enable it only if 1510 * If we are disabling tracing, then disable all records that
1056 * it's filtered 1511 * are enabled.
1057 */ 1512 */
1058 if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) { 1513 if (enable && (rec->flags & ~FTRACE_FL_MASK))
1059 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER)) 1514 flag = FTRACE_FL_ENABLED;
1060 flag = FTRACE_FL_ENABLED;
1061 }
1062 1515
1063 /* If the state of this record hasn't changed, then do nothing */ 1516 /* If the state of this record hasn't changed, then do nothing */
1064 if ((rec->flags & FTRACE_FL_ENABLED) == flag) 1517 if ((rec->flags & FTRACE_FL_ENABLED) == flag)
@@ -1079,19 +1532,16 @@ static void ftrace_replace_code(int enable)
1079 struct ftrace_page *pg; 1532 struct ftrace_page *pg;
1080 int failed; 1533 int failed;
1081 1534
1535 if (unlikely(ftrace_disabled))
1536 return;
1537
1082 do_for_each_ftrace_rec(pg, rec) { 1538 do_for_each_ftrace_rec(pg, rec) {
1083 /* 1539 /* Skip over free records */
1084 * Skip over free records, records that have 1540 if (rec->flags & FTRACE_FL_FREE)
1085 * failed and not converted.
1086 */
1087 if (rec->flags & FTRACE_FL_FREE ||
1088 rec->flags & FTRACE_FL_FAILED ||
1089 !(rec->flags & FTRACE_FL_CONVERTED))
1090 continue; 1541 continue;
1091 1542
1092 failed = __ftrace_replace_code(rec, enable); 1543 failed = __ftrace_replace_code(rec, enable);
1093 if (failed) { 1544 if (failed) {
1094 rec->flags |= FTRACE_FL_FAILED;
1095 ftrace_bug(failed, rec->ip); 1545 ftrace_bug(failed, rec->ip);
1096 /* Stop processing */ 1546 /* Stop processing */
1097 return; 1547 return;
@@ -1107,10 +1557,12 @@ ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1107 1557
1108 ip = rec->ip; 1558 ip = rec->ip;
1109 1559
1560 if (unlikely(ftrace_disabled))
1561 return 0;
1562
1110 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); 1563 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1111 if (ret) { 1564 if (ret) {
1112 ftrace_bug(ret, ip); 1565 ftrace_bug(ret, ip);
1113 rec->flags |= FTRACE_FL_FAILED;
1114 return 0; 1566 return 0;
1115 } 1567 }
1116 return 1; 1568 return 1;
@@ -1171,6 +1623,7 @@ static void ftrace_run_update_code(int command)
1171 1623
1172static ftrace_func_t saved_ftrace_func; 1624static ftrace_func_t saved_ftrace_func;
1173static int ftrace_start_up; 1625static int ftrace_start_up;
1626static int global_start_up;
1174 1627
1175static void ftrace_startup_enable(int command) 1628static void ftrace_startup_enable(int command)
1176{ 1629{
@@ -1185,19 +1638,36 @@ static void ftrace_startup_enable(int command)
1185 ftrace_run_update_code(command); 1638 ftrace_run_update_code(command);
1186} 1639}
1187 1640
1188static void ftrace_startup(int command) 1641static void ftrace_startup(struct ftrace_ops *ops, int command)
1189{ 1642{
1643 bool hash_enable = true;
1644
1190 if (unlikely(ftrace_disabled)) 1645 if (unlikely(ftrace_disabled))
1191 return; 1646 return;
1192 1647
1193 ftrace_start_up++; 1648 ftrace_start_up++;
1194 command |= FTRACE_ENABLE_CALLS; 1649 command |= FTRACE_ENABLE_CALLS;
1195 1650
1651 /* ops marked global share the filter hashes */
1652 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1653 ops = &global_ops;
1654 /* Don't update hash if global is already set */
1655 if (global_start_up)
1656 hash_enable = false;
1657 global_start_up++;
1658 }
1659
1660 ops->flags |= FTRACE_OPS_FL_ENABLED;
1661 if (hash_enable)
1662 ftrace_hash_rec_enable(ops, 1);
1663
1196 ftrace_startup_enable(command); 1664 ftrace_startup_enable(command);
1197} 1665}
1198 1666
1199static void ftrace_shutdown(int command) 1667static void ftrace_shutdown(struct ftrace_ops *ops, int command)
1200{ 1668{
1669 bool hash_disable = true;
1670
1201 if (unlikely(ftrace_disabled)) 1671 if (unlikely(ftrace_disabled))
1202 return; 1672 return;
1203 1673
@@ -1209,6 +1679,23 @@ static void ftrace_shutdown(int command)
1209 */ 1679 */
1210 WARN_ON_ONCE(ftrace_start_up < 0); 1680 WARN_ON_ONCE(ftrace_start_up < 0);
1211 1681
1682 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1683 ops = &global_ops;
1684 global_start_up--;
1685 WARN_ON_ONCE(global_start_up < 0);
1686 /* Don't update hash if global still has users */
1687 if (global_start_up) {
1688 WARN_ON_ONCE(!ftrace_start_up);
1689 hash_disable = false;
1690 }
1691 }
1692
1693 if (hash_disable)
1694 ftrace_hash_rec_disable(ops, 1);
1695
1696 if (ops != &global_ops || !global_start_up)
1697 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
1698
1212 if (!ftrace_start_up) 1699 if (!ftrace_start_up)
1213 command |= FTRACE_DISABLE_CALLS; 1700 command |= FTRACE_DISABLE_CALLS;
1214 1701
@@ -1273,10 +1760,10 @@ static int ftrace_update_code(struct module *mod)
1273 */ 1760 */
1274 if (!ftrace_code_disable(mod, p)) { 1761 if (!ftrace_code_disable(mod, p)) {
1275 ftrace_free_rec(p); 1762 ftrace_free_rec(p);
1276 continue; 1763 /* Game over */
1764 break;
1277 } 1765 }
1278 1766
1279 p->flags |= FTRACE_FL_CONVERTED;
1280 ftrace_update_cnt++; 1767 ftrace_update_cnt++;
1281 1768
1282 /* 1769 /*
@@ -1351,9 +1838,9 @@ static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1351enum { 1838enum {
1352 FTRACE_ITER_FILTER = (1 << 0), 1839 FTRACE_ITER_FILTER = (1 << 0),
1353 FTRACE_ITER_NOTRACE = (1 << 1), 1840 FTRACE_ITER_NOTRACE = (1 << 1),
1354 FTRACE_ITER_FAILURES = (1 << 2), 1841 FTRACE_ITER_PRINTALL = (1 << 2),
1355 FTRACE_ITER_PRINTALL = (1 << 3), 1842 FTRACE_ITER_HASH = (1 << 3),
1356 FTRACE_ITER_HASH = (1 << 4), 1843 FTRACE_ITER_ENABLED = (1 << 4),
1357}; 1844};
1358 1845
1359#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 1846#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
@@ -1365,6 +1852,8 @@ struct ftrace_iterator {
1365 struct dyn_ftrace *func; 1852 struct dyn_ftrace *func;
1366 struct ftrace_func_probe *probe; 1853 struct ftrace_func_probe *probe;
1367 struct trace_parser parser; 1854 struct trace_parser parser;
1855 struct ftrace_hash *hash;
1856 struct ftrace_ops *ops;
1368 int hidx; 1857 int hidx;
1369 int idx; 1858 int idx;
1370 unsigned flags; 1859 unsigned flags;
@@ -1461,8 +1950,12 @@ static void *
1461t_next(struct seq_file *m, void *v, loff_t *pos) 1950t_next(struct seq_file *m, void *v, loff_t *pos)
1462{ 1951{
1463 struct ftrace_iterator *iter = m->private; 1952 struct ftrace_iterator *iter = m->private;
1953 struct ftrace_ops *ops = &global_ops;
1464 struct dyn_ftrace *rec = NULL; 1954 struct dyn_ftrace *rec = NULL;
1465 1955
1956 if (unlikely(ftrace_disabled))
1957 return NULL;
1958
1466 if (iter->flags & FTRACE_ITER_HASH) 1959 if (iter->flags & FTRACE_ITER_HASH)
1467 return t_hash_next(m, pos); 1960 return t_hash_next(m, pos);
1468 1961
@@ -1483,17 +1976,15 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
1483 rec = &iter->pg->records[iter->idx++]; 1976 rec = &iter->pg->records[iter->idx++];
1484 if ((rec->flags & FTRACE_FL_FREE) || 1977 if ((rec->flags & FTRACE_FL_FREE) ||
1485 1978
1486 (!(iter->flags & FTRACE_ITER_FAILURES) &&
1487 (rec->flags & FTRACE_FL_FAILED)) ||
1488
1489 ((iter->flags & FTRACE_ITER_FAILURES) &&
1490 !(rec->flags & FTRACE_FL_FAILED)) ||
1491
1492 ((iter->flags & FTRACE_ITER_FILTER) && 1979 ((iter->flags & FTRACE_ITER_FILTER) &&
1493 !(rec->flags & FTRACE_FL_FILTER)) || 1980 !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
1494 1981
1495 ((iter->flags & FTRACE_ITER_NOTRACE) && 1982 ((iter->flags & FTRACE_ITER_NOTRACE) &&
1496 !(rec->flags & FTRACE_FL_NOTRACE))) { 1983 !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
1984
1985 ((iter->flags & FTRACE_ITER_ENABLED) &&
1986 !(rec->flags & ~FTRACE_FL_MASK))) {
1987
1497 rec = NULL; 1988 rec = NULL;
1498 goto retry; 1989 goto retry;
1499 } 1990 }
@@ -1517,10 +2008,15 @@ static void reset_iter_read(struct ftrace_iterator *iter)
1517static void *t_start(struct seq_file *m, loff_t *pos) 2008static void *t_start(struct seq_file *m, loff_t *pos)
1518{ 2009{
1519 struct ftrace_iterator *iter = m->private; 2010 struct ftrace_iterator *iter = m->private;
2011 struct ftrace_ops *ops = &global_ops;
1520 void *p = NULL; 2012 void *p = NULL;
1521 loff_t l; 2013 loff_t l;
1522 2014
1523 mutex_lock(&ftrace_lock); 2015 mutex_lock(&ftrace_lock);
2016
2017 if (unlikely(ftrace_disabled))
2018 return NULL;
2019
1524 /* 2020 /*
1525 * If an lseek was done, then reset and start from beginning. 2021 * If an lseek was done, then reset and start from beginning.
1526 */ 2022 */
@@ -1532,7 +2028,7 @@ static void *t_start(struct seq_file *m, loff_t *pos)
1532 * off, we can short cut and just print out that all 2028 * off, we can short cut and just print out that all
1533 * functions are enabled. 2029 * functions are enabled.
1534 */ 2030 */
1535 if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) { 2031 if (iter->flags & FTRACE_ITER_FILTER && !ops->filter_hash->count) {
1536 if (*pos > 0) 2032 if (*pos > 0)
1537 return t_hash_start(m, pos); 2033 return t_hash_start(m, pos);
1538 iter->flags |= FTRACE_ITER_PRINTALL; 2034 iter->flags |= FTRACE_ITER_PRINTALL;
@@ -1590,7 +2086,11 @@ static int t_show(struct seq_file *m, void *v)
1590 if (!rec) 2086 if (!rec)
1591 return 0; 2087 return 0;
1592 2088
1593 seq_printf(m, "%ps\n", (void *)rec->ip); 2089 seq_printf(m, "%ps", (void *)rec->ip);
2090 if (iter->flags & FTRACE_ITER_ENABLED)
2091 seq_printf(m, " (%ld)",
2092 rec->flags & ~FTRACE_FL_MASK);
2093 seq_printf(m, "\n");
1594 2094
1595 return 0; 2095 return 0;
1596} 2096}
@@ -1630,44 +2130,46 @@ ftrace_avail_open(struct inode *inode, struct file *file)
1630} 2130}
1631 2131
1632static int 2132static int
1633ftrace_failures_open(struct inode *inode, struct file *file) 2133ftrace_enabled_open(struct inode *inode, struct file *file)
1634{ 2134{
1635 int ret;
1636 struct seq_file *m;
1637 struct ftrace_iterator *iter; 2135 struct ftrace_iterator *iter;
2136 int ret;
2137
2138 if (unlikely(ftrace_disabled))
2139 return -ENODEV;
2140
2141 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2142 if (!iter)
2143 return -ENOMEM;
2144
2145 iter->pg = ftrace_pages_start;
2146 iter->flags = FTRACE_ITER_ENABLED;
1638 2147
1639 ret = ftrace_avail_open(inode, file); 2148 ret = seq_open(file, &show_ftrace_seq_ops);
1640 if (!ret) { 2149 if (!ret) {
1641 m = file->private_data; 2150 struct seq_file *m = file->private_data;
1642 iter = m->private; 2151
1643 iter->flags = FTRACE_ITER_FAILURES; 2152 m->private = iter;
2153 } else {
2154 kfree(iter);
1644 } 2155 }
1645 2156
1646 return ret; 2157 return ret;
1647} 2158}
1648 2159
1649 2160static void ftrace_filter_reset(struct ftrace_hash *hash)
1650static void ftrace_filter_reset(int enable)
1651{ 2161{
1652 struct ftrace_page *pg;
1653 struct dyn_ftrace *rec;
1654 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1655
1656 mutex_lock(&ftrace_lock); 2162 mutex_lock(&ftrace_lock);
1657 if (enable) 2163 ftrace_hash_clear(hash);
1658 ftrace_filtered = 0;
1659 do_for_each_ftrace_rec(pg, rec) {
1660 if (rec->flags & FTRACE_FL_FAILED)
1661 continue;
1662 rec->flags &= ~type;
1663 } while_for_each_ftrace_rec();
1664 mutex_unlock(&ftrace_lock); 2164 mutex_unlock(&ftrace_lock);
1665} 2165}
1666 2166
1667static int 2167static int
1668ftrace_regex_open(struct inode *inode, struct file *file, int enable) 2168ftrace_regex_open(struct ftrace_ops *ops, int flag,
2169 struct inode *inode, struct file *file)
1669{ 2170{
1670 struct ftrace_iterator *iter; 2171 struct ftrace_iterator *iter;
2172 struct ftrace_hash *hash;
1671 int ret = 0; 2173 int ret = 0;
1672 2174
1673 if (unlikely(ftrace_disabled)) 2175 if (unlikely(ftrace_disabled))
@@ -1682,21 +2184,42 @@ ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1682 return -ENOMEM; 2184 return -ENOMEM;
1683 } 2185 }
1684 2186
2187 if (flag & FTRACE_ITER_NOTRACE)
2188 hash = ops->notrace_hash;
2189 else
2190 hash = ops->filter_hash;
2191
2192 iter->ops = ops;
2193 iter->flags = flag;
2194
2195 if (file->f_mode & FMODE_WRITE) {
2196 mutex_lock(&ftrace_lock);
2197 iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
2198 mutex_unlock(&ftrace_lock);
2199
2200 if (!iter->hash) {
2201 trace_parser_put(&iter->parser);
2202 kfree(iter);
2203 return -ENOMEM;
2204 }
2205 }
2206
1685 mutex_lock(&ftrace_regex_lock); 2207 mutex_lock(&ftrace_regex_lock);
2208
1686 if ((file->f_mode & FMODE_WRITE) && 2209 if ((file->f_mode & FMODE_WRITE) &&
1687 (file->f_flags & O_TRUNC)) 2210 (file->f_flags & O_TRUNC))
1688 ftrace_filter_reset(enable); 2211 ftrace_filter_reset(iter->hash);
1689 2212
1690 if (file->f_mode & FMODE_READ) { 2213 if (file->f_mode & FMODE_READ) {
1691 iter->pg = ftrace_pages_start; 2214 iter->pg = ftrace_pages_start;
1692 iter->flags = enable ? FTRACE_ITER_FILTER :
1693 FTRACE_ITER_NOTRACE;
1694 2215
1695 ret = seq_open(file, &show_ftrace_seq_ops); 2216 ret = seq_open(file, &show_ftrace_seq_ops);
1696 if (!ret) { 2217 if (!ret) {
1697 struct seq_file *m = file->private_data; 2218 struct seq_file *m = file->private_data;
1698 m->private = iter; 2219 m->private = iter;
1699 } else { 2220 } else {
2221 /* Failed */
2222 free_ftrace_hash(iter->hash);
1700 trace_parser_put(&iter->parser); 2223 trace_parser_put(&iter->parser);
1701 kfree(iter); 2224 kfree(iter);
1702 } 2225 }
@@ -1710,13 +2233,15 @@ ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1710static int 2233static int
1711ftrace_filter_open(struct inode *inode, struct file *file) 2234ftrace_filter_open(struct inode *inode, struct file *file)
1712{ 2235{
1713 return ftrace_regex_open(inode, file, 1); 2236 return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER,
2237 inode, file);
1714} 2238}
1715 2239
1716static int 2240static int
1717ftrace_notrace_open(struct inode *inode, struct file *file) 2241ftrace_notrace_open(struct inode *inode, struct file *file)
1718{ 2242{
1719 return ftrace_regex_open(inode, file, 0); 2243 return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
2244 inode, file);
1720} 2245}
1721 2246
1722static loff_t 2247static loff_t
@@ -1761,86 +2286,99 @@ static int ftrace_match(char *str, char *regex, int len, int type)
1761} 2286}
1762 2287
1763static int 2288static int
1764ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type) 2289enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2290{
2291 struct ftrace_func_entry *entry;
2292 int ret = 0;
2293
2294 entry = ftrace_lookup_ip(hash, rec->ip);
2295 if (not) {
2296 /* Do nothing if it doesn't exist */
2297 if (!entry)
2298 return 0;
2299
2300 free_hash_entry(hash, entry);
2301 } else {
2302 /* Do nothing if it exists */
2303 if (entry)
2304 return 0;
2305
2306 ret = add_hash_entry(hash, rec->ip);
2307 }
2308 return ret;
2309}
2310
2311static int
2312ftrace_match_record(struct dyn_ftrace *rec, char *mod,
2313 char *regex, int len, int type)
1765{ 2314{
1766 char str[KSYM_SYMBOL_LEN]; 2315 char str[KSYM_SYMBOL_LEN];
2316 char *modname;
2317
2318 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
2319
2320 if (mod) {
2321 /* module lookup requires matching the module */
2322 if (!modname || strcmp(modname, mod))
2323 return 0;
2324
2325 /* blank search means to match all funcs in the mod */
2326 if (!len)
2327 return 1;
2328 }
1767 2329
1768 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1769 return ftrace_match(str, regex, len, type); 2330 return ftrace_match(str, regex, len, type);
1770} 2331}
1771 2332
1772static int ftrace_match_records(char *buff, int len, int enable) 2333static int
2334match_records(struct ftrace_hash *hash, char *buff,
2335 int len, char *mod, int not)
1773{ 2336{
1774 unsigned int search_len; 2337 unsigned search_len = 0;
1775 struct ftrace_page *pg; 2338 struct ftrace_page *pg;
1776 struct dyn_ftrace *rec; 2339 struct dyn_ftrace *rec;
1777 unsigned long flag; 2340 int type = MATCH_FULL;
1778 char *search; 2341 char *search = buff;
1779 int type;
1780 int not;
1781 int found = 0; 2342 int found = 0;
2343 int ret;
1782 2344
1783 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 2345 if (len) {
1784 type = filter_parse_regex(buff, len, &search, &not); 2346 type = filter_parse_regex(buff, len, &search, &not);
1785 2347 search_len = strlen(search);
1786 search_len = strlen(search); 2348 }
1787 2349
1788 mutex_lock(&ftrace_lock); 2350 mutex_lock(&ftrace_lock);
1789 do_for_each_ftrace_rec(pg, rec) {
1790 2351
1791 if (rec->flags & FTRACE_FL_FAILED) 2352 if (unlikely(ftrace_disabled))
1792 continue; 2353 goto out_unlock;
1793 2354
1794 if (ftrace_match_record(rec, search, search_len, type)) { 2355 do_for_each_ftrace_rec(pg, rec) {
1795 if (not) 2356
1796 rec->flags &= ~flag; 2357 if (ftrace_match_record(rec, mod, search, search_len, type)) {
1797 else 2358 ret = enter_record(hash, rec, not);
1798 rec->flags |= flag; 2359 if (ret < 0) {
2360 found = ret;
2361 goto out_unlock;
2362 }
1799 found = 1; 2363 found = 1;
1800 } 2364 }
1801 /*
1802 * Only enable filtering if we have a function that
1803 * is filtered on.
1804 */
1805 if (enable && (rec->flags & FTRACE_FL_FILTER))
1806 ftrace_filtered = 1;
1807 } while_for_each_ftrace_rec(); 2365 } while_for_each_ftrace_rec();
2366 out_unlock:
1808 mutex_unlock(&ftrace_lock); 2367 mutex_unlock(&ftrace_lock);
1809 2368
1810 return found; 2369 return found;
1811} 2370}
1812 2371
1813static int 2372static int
1814ftrace_match_module_record(struct dyn_ftrace *rec, char *mod, 2373ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
1815 char *regex, int len, int type)
1816{ 2374{
1817 char str[KSYM_SYMBOL_LEN]; 2375 return match_records(hash, buff, len, NULL, 0);
1818 char *modname;
1819
1820 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1821
1822 if (!modname || strcmp(modname, mod))
1823 return 0;
1824
1825 /* blank search means to match all funcs in the mod */
1826 if (len)
1827 return ftrace_match(str, regex, len, type);
1828 else
1829 return 1;
1830} 2376}
1831 2377
1832static int ftrace_match_module_records(char *buff, char *mod, int enable) 2378static int
2379ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
1833{ 2380{
1834 unsigned search_len = 0;
1835 struct ftrace_page *pg;
1836 struct dyn_ftrace *rec;
1837 int type = MATCH_FULL;
1838 char *search = buff;
1839 unsigned long flag;
1840 int not = 0; 2381 int not = 0;
1841 int found = 0;
1842
1843 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1844 2382
1845 /* blank or '*' mean the same */ 2383 /* blank or '*' mean the same */
1846 if (strcmp(buff, "*") == 0) 2384 if (strcmp(buff, "*") == 0)
@@ -1852,32 +2390,7 @@ static int ftrace_match_module_records(char *buff, char *mod, int enable)
1852 not = 1; 2390 not = 1;
1853 } 2391 }
1854 2392
1855 if (strlen(buff)) { 2393 return match_records(hash, buff, strlen(buff), mod, not);
1856 type = filter_parse_regex(buff, strlen(buff), &search, &not);
1857 search_len = strlen(search);
1858 }
1859
1860 mutex_lock(&ftrace_lock);
1861 do_for_each_ftrace_rec(pg, rec) {
1862
1863 if (rec->flags & FTRACE_FL_FAILED)
1864 continue;
1865
1866 if (ftrace_match_module_record(rec, mod,
1867 search, search_len, type)) {
1868 if (not)
1869 rec->flags &= ~flag;
1870 else
1871 rec->flags |= flag;
1872 found = 1;
1873 }
1874 if (enable && (rec->flags & FTRACE_FL_FILTER))
1875 ftrace_filtered = 1;
1876
1877 } while_for_each_ftrace_rec();
1878 mutex_unlock(&ftrace_lock);
1879
1880 return found;
1881} 2394}
1882 2395
1883/* 2396/*
@@ -1888,7 +2401,10 @@ static int ftrace_match_module_records(char *buff, char *mod, int enable)
1888static int 2401static int
1889ftrace_mod_callback(char *func, char *cmd, char *param, int enable) 2402ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1890{ 2403{
2404 struct ftrace_ops *ops = &global_ops;
2405 struct ftrace_hash *hash;
1891 char *mod; 2406 char *mod;
2407 int ret = -EINVAL;
1892 2408
1893 /* 2409 /*
1894 * cmd == 'mod' because we only registered this func 2410 * cmd == 'mod' because we only registered this func
@@ -1900,15 +2416,24 @@ ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1900 2416
1901 /* we must have a module name */ 2417 /* we must have a module name */
1902 if (!param) 2418 if (!param)
1903 return -EINVAL; 2419 return ret;
1904 2420
1905 mod = strsep(&param, ":"); 2421 mod = strsep(&param, ":");
1906 if (!strlen(mod)) 2422 if (!strlen(mod))
1907 return -EINVAL; 2423 return ret;
1908 2424
1909 if (ftrace_match_module_records(func, mod, enable)) 2425 if (enable)
1910 return 0; 2426 hash = ops->filter_hash;
1911 return -EINVAL; 2427 else
2428 hash = ops->notrace_hash;
2429
2430 ret = ftrace_match_module_records(hash, func, mod);
2431 if (!ret)
2432 ret = -EINVAL;
2433 if (ret < 0)
2434 return ret;
2435
2436 return 0;
1912} 2437}
1913 2438
1914static struct ftrace_func_command ftrace_mod_cmd = { 2439static struct ftrace_func_command ftrace_mod_cmd = {
@@ -1959,6 +2484,7 @@ static int ftrace_probe_registered;
1959 2484
1960static void __enable_ftrace_function_probe(void) 2485static void __enable_ftrace_function_probe(void)
1961{ 2486{
2487 int ret;
1962 int i; 2488 int i;
1963 2489
1964 if (ftrace_probe_registered) 2490 if (ftrace_probe_registered)
@@ -1973,13 +2499,16 @@ static void __enable_ftrace_function_probe(void)
1973 if (i == FTRACE_FUNC_HASHSIZE) 2499 if (i == FTRACE_FUNC_HASHSIZE)
1974 return; 2500 return;
1975 2501
1976 __register_ftrace_function(&trace_probe_ops); 2502 ret = __register_ftrace_function(&trace_probe_ops);
1977 ftrace_startup(0); 2503 if (!ret)
2504 ftrace_startup(&trace_probe_ops, 0);
2505
1978 ftrace_probe_registered = 1; 2506 ftrace_probe_registered = 1;
1979} 2507}
1980 2508
1981static void __disable_ftrace_function_probe(void) 2509static void __disable_ftrace_function_probe(void)
1982{ 2510{
2511 int ret;
1983 int i; 2512 int i;
1984 2513
1985 if (!ftrace_probe_registered) 2514 if (!ftrace_probe_registered)
@@ -1992,8 +2521,10 @@ static void __disable_ftrace_function_probe(void)
1992 } 2521 }
1993 2522
1994 /* no more funcs left */ 2523 /* no more funcs left */
1995 __unregister_ftrace_function(&trace_probe_ops); 2524 ret = __unregister_ftrace_function(&trace_probe_ops);
1996 ftrace_shutdown(0); 2525 if (!ret)
2526 ftrace_shutdown(&trace_probe_ops, 0);
2527
1997 ftrace_probe_registered = 0; 2528 ftrace_probe_registered = 0;
1998} 2529}
1999 2530
@@ -2029,12 +2560,13 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2029 return -EINVAL; 2560 return -EINVAL;
2030 2561
2031 mutex_lock(&ftrace_lock); 2562 mutex_lock(&ftrace_lock);
2032 do_for_each_ftrace_rec(pg, rec) {
2033 2563
2034 if (rec->flags & FTRACE_FL_FAILED) 2564 if (unlikely(ftrace_disabled))
2035 continue; 2565 goto out_unlock;
2566
2567 do_for_each_ftrace_rec(pg, rec) {
2036 2568
2037 if (!ftrace_match_record(rec, search, len, type)) 2569 if (!ftrace_match_record(rec, NULL, search, len, type))
2038 continue; 2570 continue;
2039 2571
2040 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 2572 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
@@ -2195,18 +2727,22 @@ int unregister_ftrace_command(struct ftrace_func_command *cmd)
2195 return ret; 2727 return ret;
2196} 2728}
2197 2729
2198static int ftrace_process_regex(char *buff, int len, int enable) 2730static int ftrace_process_regex(struct ftrace_hash *hash,
2731 char *buff, int len, int enable)
2199{ 2732{
2200 char *func, *command, *next = buff; 2733 char *func, *command, *next = buff;
2201 struct ftrace_func_command *p; 2734 struct ftrace_func_command *p;
2202 int ret = -EINVAL; 2735 int ret;
2203 2736
2204 func = strsep(&next, ":"); 2737 func = strsep(&next, ":");
2205 2738
2206 if (!next) { 2739 if (!next) {
2207 if (ftrace_match_records(func, len, enable)) 2740 ret = ftrace_match_records(hash, func, len);
2208 return 0; 2741 if (!ret)
2209 return ret; 2742 ret = -EINVAL;
2743 if (ret < 0)
2744 return ret;
2745 return 0;
2210 } 2746 }
2211 2747
2212 /* command found */ 2748 /* command found */
@@ -2239,6 +2775,10 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
2239 2775
2240 mutex_lock(&ftrace_regex_lock); 2776 mutex_lock(&ftrace_regex_lock);
2241 2777
2778 ret = -ENODEV;
2779 if (unlikely(ftrace_disabled))
2780 goto out_unlock;
2781
2242 if (file->f_mode & FMODE_READ) { 2782 if (file->f_mode & FMODE_READ) {
2243 struct seq_file *m = file->private_data; 2783 struct seq_file *m = file->private_data;
2244 iter = m->private; 2784 iter = m->private;
@@ -2250,7 +2790,7 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
2250 2790
2251 if (read >= 0 && trace_parser_loaded(parser) && 2791 if (read >= 0 && trace_parser_loaded(parser) &&
2252 !trace_parser_cont(parser)) { 2792 !trace_parser_cont(parser)) {
2253 ret = ftrace_process_regex(parser->buffer, 2793 ret = ftrace_process_regex(iter->hash, parser->buffer,
2254 parser->idx, enable); 2794 parser->idx, enable);
2255 trace_parser_clear(parser); 2795 trace_parser_clear(parser);
2256 if (ret) 2796 if (ret)
@@ -2278,22 +2818,49 @@ ftrace_notrace_write(struct file *file, const char __user *ubuf,
2278 return ftrace_regex_write(file, ubuf, cnt, ppos, 0); 2818 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2279} 2819}
2280 2820
2281static void 2821static int
2282ftrace_set_regex(unsigned char *buf, int len, int reset, int enable) 2822ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
2823 int reset, int enable)
2283{ 2824{
2825 struct ftrace_hash **orig_hash;
2826 struct ftrace_hash *hash;
2827 int ret;
2828
2829 /* All global ops uses the global ops filters */
2830 if (ops->flags & FTRACE_OPS_FL_GLOBAL)
2831 ops = &global_ops;
2832
2284 if (unlikely(ftrace_disabled)) 2833 if (unlikely(ftrace_disabled))
2285 return; 2834 return -ENODEV;
2835
2836 if (enable)
2837 orig_hash = &ops->filter_hash;
2838 else
2839 orig_hash = &ops->notrace_hash;
2840
2841 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
2842 if (!hash)
2843 return -ENOMEM;
2286 2844
2287 mutex_lock(&ftrace_regex_lock); 2845 mutex_lock(&ftrace_regex_lock);
2288 if (reset) 2846 if (reset)
2289 ftrace_filter_reset(enable); 2847 ftrace_filter_reset(hash);
2290 if (buf) 2848 if (buf)
2291 ftrace_match_records(buf, len, enable); 2849 ftrace_match_records(hash, buf, len);
2850
2851 mutex_lock(&ftrace_lock);
2852 ret = ftrace_hash_move(orig_hash, hash);
2853 mutex_unlock(&ftrace_lock);
2854
2292 mutex_unlock(&ftrace_regex_lock); 2855 mutex_unlock(&ftrace_regex_lock);
2856
2857 free_ftrace_hash(hash);
2858 return ret;
2293} 2859}
2294 2860
2295/** 2861/**
2296 * ftrace_set_filter - set a function to filter on in ftrace 2862 * ftrace_set_filter - set a function to filter on in ftrace
2863 * @ops - the ops to set the filter with
2297 * @buf - the string that holds the function filter text. 2864 * @buf - the string that holds the function filter text.
2298 * @len - the length of the string. 2865 * @len - the length of the string.
2299 * @reset - non zero to reset all filters before applying this filter. 2866 * @reset - non zero to reset all filters before applying this filter.
@@ -2301,13 +2868,16 @@ ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2301 * Filters denote which functions should be enabled when tracing is enabled. 2868 * Filters denote which functions should be enabled when tracing is enabled.
2302 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 2869 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2303 */ 2870 */
2304void ftrace_set_filter(unsigned char *buf, int len, int reset) 2871void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
2872 int len, int reset)
2305{ 2873{
2306 ftrace_set_regex(buf, len, reset, 1); 2874 ftrace_set_regex(ops, buf, len, reset, 1);
2307} 2875}
2876EXPORT_SYMBOL_GPL(ftrace_set_filter);
2308 2877
2309/** 2878/**
2310 * ftrace_set_notrace - set a function to not trace in ftrace 2879 * ftrace_set_notrace - set a function to not trace in ftrace
2880 * @ops - the ops to set the notrace filter with
2311 * @buf - the string that holds the function notrace text. 2881 * @buf - the string that holds the function notrace text.
2312 * @len - the length of the string. 2882 * @len - the length of the string.
2313 * @reset - non zero to reset all filters before applying this filter. 2883 * @reset - non zero to reset all filters before applying this filter.
@@ -2316,10 +2886,44 @@ void ftrace_set_filter(unsigned char *buf, int len, int reset)
2316 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 2886 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2317 * for tracing. 2887 * for tracing.
2318 */ 2888 */
2319void ftrace_set_notrace(unsigned char *buf, int len, int reset) 2889void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
2890 int len, int reset)
2320{ 2891{
2321 ftrace_set_regex(buf, len, reset, 0); 2892 ftrace_set_regex(ops, buf, len, reset, 0);
2322} 2893}
2894EXPORT_SYMBOL_GPL(ftrace_set_notrace);
2895/**
2896 * ftrace_set_filter - set a function to filter on in ftrace
2897 * @ops - the ops to set the filter with
2898 * @buf - the string that holds the function filter text.
2899 * @len - the length of the string.
2900 * @reset - non zero to reset all filters before applying this filter.
2901 *
2902 * Filters denote which functions should be enabled when tracing is enabled.
2903 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2904 */
2905void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
2906{
2907 ftrace_set_regex(&global_ops, buf, len, reset, 1);
2908}
2909EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
2910
2911/**
2912 * ftrace_set_notrace - set a function to not trace in ftrace
2913 * @ops - the ops to set the notrace filter with
2914 * @buf - the string that holds the function notrace text.
2915 * @len - the length of the string.
2916 * @reset - non zero to reset all filters before applying this filter.
2917 *
2918 * Notrace Filters denote which functions should not be enabled when tracing
2919 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2920 * for tracing.
2921 */
2922void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
2923{
2924 ftrace_set_regex(&global_ops, buf, len, reset, 0);
2925}
2926EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
2323 2927
2324/* 2928/*
2325 * command line interface to allow users to set filters on boot up. 2929 * command line interface to allow users to set filters on boot up.
@@ -2370,22 +2974,23 @@ static void __init set_ftrace_early_graph(char *buf)
2370} 2974}
2371#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2975#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2372 2976
2373static void __init set_ftrace_early_filter(char *buf, int enable) 2977static void __init
2978set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable)
2374{ 2979{
2375 char *func; 2980 char *func;
2376 2981
2377 while (buf) { 2982 while (buf) {
2378 func = strsep(&buf, ","); 2983 func = strsep(&buf, ",");
2379 ftrace_set_regex(func, strlen(func), 0, enable); 2984 ftrace_set_regex(ops, func, strlen(func), 0, enable);
2380 } 2985 }
2381} 2986}
2382 2987
2383static void __init set_ftrace_early_filters(void) 2988static void __init set_ftrace_early_filters(void)
2384{ 2989{
2385 if (ftrace_filter_buf[0]) 2990 if (ftrace_filter_buf[0])
2386 set_ftrace_early_filter(ftrace_filter_buf, 1); 2991 set_ftrace_early_filter(&global_ops, ftrace_filter_buf, 1);
2387 if (ftrace_notrace_buf[0]) 2992 if (ftrace_notrace_buf[0])
2388 set_ftrace_early_filter(ftrace_notrace_buf, 0); 2993 set_ftrace_early_filter(&global_ops, ftrace_notrace_buf, 0);
2389#ifdef CONFIG_FUNCTION_GRAPH_TRACER 2994#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2390 if (ftrace_graph_buf[0]) 2995 if (ftrace_graph_buf[0])
2391 set_ftrace_early_graph(ftrace_graph_buf); 2996 set_ftrace_early_graph(ftrace_graph_buf);
@@ -2393,11 +2998,14 @@ static void __init set_ftrace_early_filters(void)
2393} 2998}
2394 2999
2395static int 3000static int
2396ftrace_regex_release(struct inode *inode, struct file *file, int enable) 3001ftrace_regex_release(struct inode *inode, struct file *file)
2397{ 3002{
2398 struct seq_file *m = (struct seq_file *)file->private_data; 3003 struct seq_file *m = (struct seq_file *)file->private_data;
2399 struct ftrace_iterator *iter; 3004 struct ftrace_iterator *iter;
3005 struct ftrace_hash **orig_hash;
2400 struct trace_parser *parser; 3006 struct trace_parser *parser;
3007 int filter_hash;
3008 int ret;
2401 3009
2402 mutex_lock(&ftrace_regex_lock); 3010 mutex_lock(&ftrace_regex_lock);
2403 if (file->f_mode & FMODE_READ) { 3011 if (file->f_mode & FMODE_READ) {
@@ -2410,33 +3018,41 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2410 parser = &iter->parser; 3018 parser = &iter->parser;
2411 if (trace_parser_loaded(parser)) { 3019 if (trace_parser_loaded(parser)) {
2412 parser->buffer[parser->idx] = 0; 3020 parser->buffer[parser->idx] = 0;
2413 ftrace_match_records(parser->buffer, parser->idx, enable); 3021 ftrace_match_records(iter->hash, parser->buffer, parser->idx);
2414 } 3022 }
2415 3023
2416 mutex_lock(&ftrace_lock);
2417 if (ftrace_start_up && ftrace_enabled)
2418 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2419 mutex_unlock(&ftrace_lock);
2420
2421 trace_parser_put(parser); 3024 trace_parser_put(parser);
3025
3026 if (file->f_mode & FMODE_WRITE) {
3027 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3028
3029 if (filter_hash)
3030 orig_hash = &iter->ops->filter_hash;
3031 else
3032 orig_hash = &iter->ops->notrace_hash;
3033
3034 mutex_lock(&ftrace_lock);
3035 /*
3036 * Remove the current set, update the hash and add
3037 * them back.
3038 */
3039 ftrace_hash_rec_disable(iter->ops, filter_hash);
3040 ret = ftrace_hash_move(orig_hash, iter->hash);
3041 if (!ret) {
3042 ftrace_hash_rec_enable(iter->ops, filter_hash);
3043 if (iter->ops->flags & FTRACE_OPS_FL_ENABLED
3044 && ftrace_enabled)
3045 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
3046 }
3047 mutex_unlock(&ftrace_lock);
3048 }
3049 free_ftrace_hash(iter->hash);
2422 kfree(iter); 3050 kfree(iter);
2423 3051
2424 mutex_unlock(&ftrace_regex_lock); 3052 mutex_unlock(&ftrace_regex_lock);
2425 return 0; 3053 return 0;
2426} 3054}
2427 3055
2428static int
2429ftrace_filter_release(struct inode *inode, struct file *file)
2430{
2431 return ftrace_regex_release(inode, file, 1);
2432}
2433
2434static int
2435ftrace_notrace_release(struct inode *inode, struct file *file)
2436{
2437 return ftrace_regex_release(inode, file, 0);
2438}
2439
2440static const struct file_operations ftrace_avail_fops = { 3056static const struct file_operations ftrace_avail_fops = {
2441 .open = ftrace_avail_open, 3057 .open = ftrace_avail_open,
2442 .read = seq_read, 3058 .read = seq_read,
@@ -2444,8 +3060,8 @@ static const struct file_operations ftrace_avail_fops = {
2444 .release = seq_release_private, 3060 .release = seq_release_private,
2445}; 3061};
2446 3062
2447static const struct file_operations ftrace_failures_fops = { 3063static const struct file_operations ftrace_enabled_fops = {
2448 .open = ftrace_failures_open, 3064 .open = ftrace_enabled_open,
2449 .read = seq_read, 3065 .read = seq_read,
2450 .llseek = seq_lseek, 3066 .llseek = seq_lseek,
2451 .release = seq_release_private, 3067 .release = seq_release_private,
@@ -2456,7 +3072,7 @@ static const struct file_operations ftrace_filter_fops = {
2456 .read = seq_read, 3072 .read = seq_read,
2457 .write = ftrace_filter_write, 3073 .write = ftrace_filter_write,
2458 .llseek = ftrace_regex_lseek, 3074 .llseek = ftrace_regex_lseek,
2459 .release = ftrace_filter_release, 3075 .release = ftrace_regex_release,
2460}; 3076};
2461 3077
2462static const struct file_operations ftrace_notrace_fops = { 3078static const struct file_operations ftrace_notrace_fops = {
@@ -2464,7 +3080,7 @@ static const struct file_operations ftrace_notrace_fops = {
2464 .read = seq_read, 3080 .read = seq_read,
2465 .write = ftrace_notrace_write, 3081 .write = ftrace_notrace_write,
2466 .llseek = ftrace_regex_lseek, 3082 .llseek = ftrace_regex_lseek,
2467 .release = ftrace_notrace_release, 3083 .release = ftrace_regex_release,
2468}; 3084};
2469 3085
2470#ifdef CONFIG_FUNCTION_GRAPH_TRACER 3086#ifdef CONFIG_FUNCTION_GRAPH_TRACER
@@ -2573,9 +3189,6 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2573 bool exists; 3189 bool exists;
2574 int i; 3190 int i;
2575 3191
2576 if (ftrace_disabled)
2577 return -ENODEV;
2578
2579 /* decode regex */ 3192 /* decode regex */
2580 type = filter_parse_regex(buffer, strlen(buffer), &search, &not); 3193 type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
2581 if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS) 3194 if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
@@ -2584,12 +3197,18 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2584 search_len = strlen(search); 3197 search_len = strlen(search);
2585 3198
2586 mutex_lock(&ftrace_lock); 3199 mutex_lock(&ftrace_lock);
3200
3201 if (unlikely(ftrace_disabled)) {
3202 mutex_unlock(&ftrace_lock);
3203 return -ENODEV;
3204 }
3205
2587 do_for_each_ftrace_rec(pg, rec) { 3206 do_for_each_ftrace_rec(pg, rec) {
2588 3207
2589 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) 3208 if (rec->flags & FTRACE_FL_FREE)
2590 continue; 3209 continue;
2591 3210
2592 if (ftrace_match_record(rec, search, search_len, type)) { 3211 if (ftrace_match_record(rec, NULL, search, search_len, type)) {
2593 /* if it is in the array */ 3212 /* if it is in the array */
2594 exists = false; 3213 exists = false;
2595 for (i = 0; i < *idx; i++) { 3214 for (i = 0; i < *idx; i++) {
@@ -2679,8 +3298,8 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2679 trace_create_file("available_filter_functions", 0444, 3298 trace_create_file("available_filter_functions", 0444,
2680 d_tracer, NULL, &ftrace_avail_fops); 3299 d_tracer, NULL, &ftrace_avail_fops);
2681 3300
2682 trace_create_file("failures", 0444, 3301 trace_create_file("enabled_functions", 0444,
2683 d_tracer, NULL, &ftrace_failures_fops); 3302 d_tracer, NULL, &ftrace_enabled_fops);
2684 3303
2685 trace_create_file("set_ftrace_filter", 0644, d_tracer, 3304 trace_create_file("set_ftrace_filter", 0644, d_tracer,
2686 NULL, &ftrace_filter_fops); 3305 NULL, &ftrace_filter_fops);
@@ -2703,7 +3322,6 @@ static int ftrace_process_locs(struct module *mod,
2703{ 3322{
2704 unsigned long *p; 3323 unsigned long *p;
2705 unsigned long addr; 3324 unsigned long addr;
2706 unsigned long flags;
2707 3325
2708 mutex_lock(&ftrace_lock); 3326 mutex_lock(&ftrace_lock);
2709 p = start; 3327 p = start;
@@ -2720,10 +3338,7 @@ static int ftrace_process_locs(struct module *mod,
2720 ftrace_record_ip(addr); 3338 ftrace_record_ip(addr);
2721 } 3339 }
2722 3340
2723 /* disable interrupts to prevent kstop machine */
2724 local_irq_save(flags);
2725 ftrace_update_code(mod); 3341 ftrace_update_code(mod);
2726 local_irq_restore(flags);
2727 mutex_unlock(&ftrace_lock); 3342 mutex_unlock(&ftrace_lock);
2728 3343
2729 return 0; 3344 return 0;
@@ -2735,10 +3350,11 @@ void ftrace_release_mod(struct module *mod)
2735 struct dyn_ftrace *rec; 3350 struct dyn_ftrace *rec;
2736 struct ftrace_page *pg; 3351 struct ftrace_page *pg;
2737 3352
3353 mutex_lock(&ftrace_lock);
3354
2738 if (ftrace_disabled) 3355 if (ftrace_disabled)
2739 return; 3356 goto out_unlock;
2740 3357
2741 mutex_lock(&ftrace_lock);
2742 do_for_each_ftrace_rec(pg, rec) { 3358 do_for_each_ftrace_rec(pg, rec) {
2743 if (within_module_core(rec->ip, mod)) { 3359 if (within_module_core(rec->ip, mod)) {
2744 /* 3360 /*
@@ -2749,6 +3365,7 @@ void ftrace_release_mod(struct module *mod)
2749 ftrace_free_rec(rec); 3365 ftrace_free_rec(rec);
2750 } 3366 }
2751 } while_for_each_ftrace_rec(); 3367 } while_for_each_ftrace_rec();
3368 out_unlock:
2752 mutex_unlock(&ftrace_lock); 3369 mutex_unlock(&ftrace_lock);
2753} 3370}
2754 3371
@@ -2835,6 +3452,10 @@ void __init ftrace_init(void)
2835 3452
2836#else 3453#else
2837 3454
3455static struct ftrace_ops global_ops = {
3456 .func = ftrace_stub,
3457};
3458
2838static int __init ftrace_nodyn_init(void) 3459static int __init ftrace_nodyn_init(void)
2839{ 3460{
2840 ftrace_enabled = 1; 3461 ftrace_enabled = 1;
@@ -2845,12 +3466,38 @@ device_initcall(ftrace_nodyn_init);
2845static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; } 3466static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2846static inline void ftrace_startup_enable(int command) { } 3467static inline void ftrace_startup_enable(int command) { }
2847/* Keep as macros so we do not need to define the commands */ 3468/* Keep as macros so we do not need to define the commands */
2848# define ftrace_startup(command) do { } while (0) 3469# define ftrace_startup(ops, command) do { } while (0)
2849# define ftrace_shutdown(command) do { } while (0) 3470# define ftrace_shutdown(ops, command) do { } while (0)
2850# define ftrace_startup_sysctl() do { } while (0) 3471# define ftrace_startup_sysctl() do { } while (0)
2851# define ftrace_shutdown_sysctl() do { } while (0) 3472# define ftrace_shutdown_sysctl() do { } while (0)
3473
3474static inline int
3475ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
3476{
3477 return 1;
3478}
3479
2852#endif /* CONFIG_DYNAMIC_FTRACE */ 3480#endif /* CONFIG_DYNAMIC_FTRACE */
2853 3481
3482static void
3483ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
3484{
3485 struct ftrace_ops *op;
3486
3487 /*
3488 * Some of the ops may be dynamically allocated,
3489 * they must be freed after a synchronize_sched().
3490 */
3491 preempt_disable_notrace();
3492 op = rcu_dereference_raw(ftrace_ops_list);
3493 while (op != &ftrace_list_end) {
3494 if (ftrace_ops_test(op, ip))
3495 op->func(ip, parent_ip);
3496 op = rcu_dereference_raw(op->next);
3497 };
3498 preempt_enable_notrace();
3499}
3500
2854static void clear_ftrace_swapper(void) 3501static void clear_ftrace_swapper(void)
2855{ 3502{
2856 struct task_struct *p; 3503 struct task_struct *p;
@@ -3143,19 +3790,23 @@ void ftrace_kill(void)
3143 */ 3790 */
3144int register_ftrace_function(struct ftrace_ops *ops) 3791int register_ftrace_function(struct ftrace_ops *ops)
3145{ 3792{
3146 int ret; 3793 int ret = -1;
3147
3148 if (unlikely(ftrace_disabled))
3149 return -1;
3150 3794
3151 mutex_lock(&ftrace_lock); 3795 mutex_lock(&ftrace_lock);
3152 3796
3797 if (unlikely(ftrace_disabled))
3798 goto out_unlock;
3799
3153 ret = __register_ftrace_function(ops); 3800 ret = __register_ftrace_function(ops);
3154 ftrace_startup(0); 3801 if (!ret)
3802 ftrace_startup(ops, 0);
3155 3803
3804
3805 out_unlock:
3156 mutex_unlock(&ftrace_lock); 3806 mutex_unlock(&ftrace_lock);
3157 return ret; 3807 return ret;
3158} 3808}
3809EXPORT_SYMBOL_GPL(register_ftrace_function);
3159 3810
3160/** 3811/**
3161 * unregister_ftrace_function - unregister a function for profiling. 3812 * unregister_ftrace_function - unregister a function for profiling.
@@ -3169,25 +3820,27 @@ int unregister_ftrace_function(struct ftrace_ops *ops)
3169 3820
3170 mutex_lock(&ftrace_lock); 3821 mutex_lock(&ftrace_lock);
3171 ret = __unregister_ftrace_function(ops); 3822 ret = __unregister_ftrace_function(ops);
3172 ftrace_shutdown(0); 3823 if (!ret)
3824 ftrace_shutdown(ops, 0);
3173 mutex_unlock(&ftrace_lock); 3825 mutex_unlock(&ftrace_lock);
3174 3826
3175 return ret; 3827 return ret;
3176} 3828}
3829EXPORT_SYMBOL_GPL(unregister_ftrace_function);
3177 3830
3178int 3831int
3179ftrace_enable_sysctl(struct ctl_table *table, int write, 3832ftrace_enable_sysctl(struct ctl_table *table, int write,
3180 void __user *buffer, size_t *lenp, 3833 void __user *buffer, size_t *lenp,
3181 loff_t *ppos) 3834 loff_t *ppos)
3182{ 3835{
3183 int ret; 3836 int ret = -ENODEV;
3184
3185 if (unlikely(ftrace_disabled))
3186 return -ENODEV;
3187 3837
3188 mutex_lock(&ftrace_lock); 3838 mutex_lock(&ftrace_lock);
3189 3839
3190 ret = proc_dointvec(table, write, buffer, lenp, ppos); 3840 if (unlikely(ftrace_disabled))
3841 goto out;
3842
3843 ret = proc_dointvec(table, write, buffer, lenp, ppos);
3191 3844
3192 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) 3845 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3193 goto out; 3846 goto out;
@@ -3199,11 +3852,11 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
3199 ftrace_startup_sysctl(); 3852 ftrace_startup_sysctl();
3200 3853
3201 /* we are starting ftrace again */ 3854 /* we are starting ftrace again */
3202 if (ftrace_list != &ftrace_list_end) { 3855 if (ftrace_ops_list != &ftrace_list_end) {
3203 if (ftrace_list->next == &ftrace_list_end) 3856 if (ftrace_ops_list->next == &ftrace_list_end)
3204 ftrace_trace_function = ftrace_list->func; 3857 ftrace_trace_function = ftrace_ops_list->func;
3205 else 3858 else
3206 ftrace_trace_function = ftrace_list_func; 3859 ftrace_trace_function = ftrace_ops_list_func;
3207 } 3860 }
3208 3861
3209 } else { 3862 } else {
@@ -3392,7 +4045,7 @@ int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3392 ftrace_graph_return = retfunc; 4045 ftrace_graph_return = retfunc;
3393 ftrace_graph_entry = entryfunc; 4046 ftrace_graph_entry = entryfunc;
3394 4047
3395 ftrace_startup(FTRACE_START_FUNC_RET); 4048 ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
3396 4049
3397out: 4050out:
3398 mutex_unlock(&ftrace_lock); 4051 mutex_unlock(&ftrace_lock);
@@ -3409,7 +4062,7 @@ void unregister_ftrace_graph(void)
3409 ftrace_graph_active--; 4062 ftrace_graph_active--;
3410 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 4063 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3411 ftrace_graph_entry = ftrace_graph_entry_stub; 4064 ftrace_graph_entry = ftrace_graph_entry_stub;
3412 ftrace_shutdown(FTRACE_STOP_FUNC_RET); 4065 ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
3413 unregister_pm_notifier(&ftrace_suspend_notifier); 4066 unregister_pm_notifier(&ftrace_suspend_notifier);
3414 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); 4067 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
3415 4068
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 1cb49be7c7fb..ee9c921d7f21 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -2014,9 +2014,10 @@ enum print_line_t print_trace_line(struct trace_iterator *iter)
2014{ 2014{
2015 enum print_line_t ret; 2015 enum print_line_t ret;
2016 2016
2017 if (iter->lost_events) 2017 if (iter->lost_events &&
2018 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 2018 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2019 iter->cpu, iter->lost_events); 2019 iter->cpu, iter->lost_events))
2020 return TRACE_TYPE_PARTIAL_LINE;
2020 2021
2021 if (iter->trace && iter->trace->print_line) { 2022 if (iter->trace && iter->trace->print_line) {
2022 ret = iter->trace->print_line(iter); 2023 ret = iter->trace->print_line(iter);
@@ -3230,6 +3231,14 @@ waitagain:
3230 3231
3231 if (iter->seq.len >= cnt) 3232 if (iter->seq.len >= cnt)
3232 break; 3233 break;
3234
3235 /*
3236 * Setting the full flag means we reached the trace_seq buffer
3237 * size and we should leave by partial output condition above.
3238 * One of the trace_seq_* functions is not used properly.
3239 */
3240 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3241 iter->ent->type);
3233 } 3242 }
3234 trace_access_unlock(iter->cpu_file); 3243 trace_access_unlock(iter->cpu_file);
3235 trace_event_read_unlock(); 3244 trace_event_read_unlock();
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 5e9dfc6286dd..6b69c4bd306f 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -419,6 +419,8 @@ extern void trace_find_cmdline(int pid, char comm[]);
419extern unsigned long ftrace_update_tot_cnt; 419extern unsigned long ftrace_update_tot_cnt;
420#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func 420#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
421extern int DYN_FTRACE_TEST_NAME(void); 421extern int DYN_FTRACE_TEST_NAME(void);
422#define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2
423extern int DYN_FTRACE_TEST_NAME2(void);
422#endif 424#endif
423 425
424extern int ring_buffer_expanded; 426extern int ring_buffer_expanded;
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 16aee4d44e8f..8d0e1cc4e974 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -149,11 +149,13 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip)
149static struct ftrace_ops trace_ops __read_mostly = 149static struct ftrace_ops trace_ops __read_mostly =
150{ 150{
151 .func = function_trace_call, 151 .func = function_trace_call,
152 .flags = FTRACE_OPS_FL_GLOBAL,
152}; 153};
153 154
154static struct ftrace_ops trace_stack_ops __read_mostly = 155static struct ftrace_ops trace_stack_ops __read_mostly =
155{ 156{
156 .func = function_stack_trace_call, 157 .func = function_stack_trace_call,
158 .flags = FTRACE_OPS_FL_GLOBAL,
157}; 159};
158 160
159/* Our two options */ 161/* Our two options */
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index a4969b47afc1..c77424be284d 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -153,6 +153,7 @@ irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
153static struct ftrace_ops trace_ops __read_mostly = 153static struct ftrace_ops trace_ops __read_mostly =
154{ 154{
155 .func = irqsoff_tracer_call, 155 .func = irqsoff_tracer_call,
156 .flags = FTRACE_OPS_FL_GLOBAL,
156}; 157};
157#endif /* CONFIG_FUNCTION_TRACER */ 158#endif /* CONFIG_FUNCTION_TRACER */
158 159
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index 456be9063c2d..cf535ccedc86 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -830,6 +830,9 @@ EXPORT_SYMBOL_GPL(unregister_ftrace_event);
830enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags, 830enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags,
831 struct trace_event *event) 831 struct trace_event *event)
832{ 832{
833 if (!trace_seq_printf(&iter->seq, "type: %d\n", iter->ent->type))
834 return TRACE_TYPE_PARTIAL_LINE;
835
833 return TRACE_TYPE_HANDLED; 836 return TRACE_TYPE_HANDLED;
834} 837}
835 838
diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c
index 2547d8813cf0..dff763b7baf1 100644
--- a/kernel/trace/trace_printk.c
+++ b/kernel/trace/trace_printk.c
@@ -32,7 +32,7 @@ static DEFINE_MUTEX(btrace_mutex);
32 32
33struct trace_bprintk_fmt { 33struct trace_bprintk_fmt {
34 struct list_head list; 34 struct list_head list;
35 char fmt[0]; 35 const char *fmt;
36}; 36};
37 37
38static inline struct trace_bprintk_fmt *lookup_format(const char *fmt) 38static inline struct trace_bprintk_fmt *lookup_format(const char *fmt)
@@ -49,6 +49,7 @@ static
49void hold_module_trace_bprintk_format(const char **start, const char **end) 49void hold_module_trace_bprintk_format(const char **start, const char **end)
50{ 50{
51 const char **iter; 51 const char **iter;
52 char *fmt;
52 53
53 mutex_lock(&btrace_mutex); 54 mutex_lock(&btrace_mutex);
54 for (iter = start; iter < end; iter++) { 55 for (iter = start; iter < end; iter++) {
@@ -58,14 +59,18 @@ void hold_module_trace_bprintk_format(const char **start, const char **end)
58 continue; 59 continue;
59 } 60 }
60 61
61 tb_fmt = kmalloc(offsetof(struct trace_bprintk_fmt, fmt) 62 tb_fmt = kmalloc(sizeof(*tb_fmt), GFP_KERNEL);
62 + strlen(*iter) + 1, GFP_KERNEL); 63 if (tb_fmt)
63 if (tb_fmt) { 64 fmt = kmalloc(strlen(*iter) + 1, GFP_KERNEL);
65 if (tb_fmt && fmt) {
64 list_add_tail(&tb_fmt->list, &trace_bprintk_fmt_list); 66 list_add_tail(&tb_fmt->list, &trace_bprintk_fmt_list);
65 strcpy(tb_fmt->fmt, *iter); 67 strcpy(fmt, *iter);
68 tb_fmt->fmt = fmt;
66 *iter = tb_fmt->fmt; 69 *iter = tb_fmt->fmt;
67 } else 70 } else {
71 kfree(tb_fmt);
68 *iter = NULL; 72 *iter = NULL;
73 }
69 } 74 }
70 mutex_unlock(&btrace_mutex); 75 mutex_unlock(&btrace_mutex);
71} 76}
@@ -84,6 +89,76 @@ static int module_trace_bprintk_format_notify(struct notifier_block *self,
84 return 0; 89 return 0;
85} 90}
86 91
92/*
93 * The debugfs/tracing/printk_formats file maps the addresses with
94 * the ASCII formats that are used in the bprintk events in the
95 * buffer. For userspace tools to be able to decode the events from
96 * the buffer, they need to be able to map the address with the format.
97 *
98 * The addresses of the bprintk formats are in their own section
99 * __trace_printk_fmt. But for modules we copy them into a link list.
100 * The code to print the formats and their addresses passes around the
101 * address of the fmt string. If the fmt address passed into the seq
102 * functions is within the kernel core __trace_printk_fmt section, then
103 * it simply uses the next pointer in the list.
104 *
105 * When the fmt pointer is outside the kernel core __trace_printk_fmt
106 * section, then we need to read the link list pointers. The trick is
107 * we pass the address of the string to the seq function just like
108 * we do for the kernel core formats. To get back the structure that
109 * holds the format, we simply use containerof() and then go to the
110 * next format in the list.
111 */
112static const char **
113find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos)
114{
115 struct trace_bprintk_fmt *mod_fmt;
116
117 if (list_empty(&trace_bprintk_fmt_list))
118 return NULL;
119
120 /*
121 * v will point to the address of the fmt record from t_next
122 * v will be NULL from t_start.
123 * If this is the first pointer or called from start
124 * then we need to walk the list.
125 */
126 if (!v || start_index == *pos) {
127 struct trace_bprintk_fmt *p;
128
129 /* search the module list */
130 list_for_each_entry(p, &trace_bprintk_fmt_list, list) {
131 if (start_index == *pos)
132 return &p->fmt;
133 start_index++;
134 }
135 /* pos > index */
136 return NULL;
137 }
138
139 /*
140 * v points to the address of the fmt field in the mod list
141 * structure that holds the module print format.
142 */
143 mod_fmt = container_of(v, typeof(*mod_fmt), fmt);
144 if (mod_fmt->list.next == &trace_bprintk_fmt_list)
145 return NULL;
146
147 mod_fmt = container_of(mod_fmt->list.next, typeof(*mod_fmt), list);
148
149 return &mod_fmt->fmt;
150}
151
152static void format_mod_start(void)
153{
154 mutex_lock(&btrace_mutex);
155}
156
157static void format_mod_stop(void)
158{
159 mutex_unlock(&btrace_mutex);
160}
161
87#else /* !CONFIG_MODULES */ 162#else /* !CONFIG_MODULES */
88__init static int 163__init static int
89module_trace_bprintk_format_notify(struct notifier_block *self, 164module_trace_bprintk_format_notify(struct notifier_block *self,
@@ -91,6 +166,13 @@ module_trace_bprintk_format_notify(struct notifier_block *self,
91{ 166{
92 return 0; 167 return 0;
93} 168}
169static inline const char **
170find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos)
171{
172 return NULL;
173}
174static inline void format_mod_start(void) { }
175static inline void format_mod_stop(void) { }
94#endif /* CONFIG_MODULES */ 176#endif /* CONFIG_MODULES */
95 177
96 178
@@ -153,20 +235,33 @@ int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap)
153} 235}
154EXPORT_SYMBOL_GPL(__ftrace_vprintk); 236EXPORT_SYMBOL_GPL(__ftrace_vprintk);
155 237
238static const char **find_next(void *v, loff_t *pos)
239{
240 const char **fmt = v;
241 int start_index;
242
243 if (!fmt)
244 fmt = __start___trace_bprintk_fmt + *pos;
245
246 start_index = __stop___trace_bprintk_fmt - __start___trace_bprintk_fmt;
247
248 if (*pos < start_index)
249 return fmt;
250
251 return find_next_mod_format(start_index, v, fmt, pos);
252}
253
156static void * 254static void *
157t_start(struct seq_file *m, loff_t *pos) 255t_start(struct seq_file *m, loff_t *pos)
158{ 256{
159 const char **fmt = __start___trace_bprintk_fmt + *pos; 257 format_mod_start();
160 258 return find_next(NULL, pos);
161 if ((unsigned long)fmt >= (unsigned long)__stop___trace_bprintk_fmt)
162 return NULL;
163 return fmt;
164} 259}
165 260
166static void *t_next(struct seq_file *m, void * v, loff_t *pos) 261static void *t_next(struct seq_file *m, void * v, loff_t *pos)
167{ 262{
168 (*pos)++; 263 (*pos)++;
169 return t_start(m, pos); 264 return find_next(v, pos);
170} 265}
171 266
172static int t_show(struct seq_file *m, void *v) 267static int t_show(struct seq_file *m, void *v)
@@ -205,6 +300,7 @@ static int t_show(struct seq_file *m, void *v)
205 300
206static void t_stop(struct seq_file *m, void *p) 301static void t_stop(struct seq_file *m, void *p)
207{ 302{
303 format_mod_stop();
208} 304}
209 305
210static const struct seq_operations show_format_seq_ops = { 306static const struct seq_operations show_format_seq_ops = {
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 7319559ed59f..f029dd4fd2ca 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -129,6 +129,7 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
129static struct ftrace_ops trace_ops __read_mostly = 129static struct ftrace_ops trace_ops __read_mostly =
130{ 130{
131 .func = wakeup_tracer_call, 131 .func = wakeup_tracer_call,
132 .flags = FTRACE_OPS_FL_GLOBAL,
132}; 133};
133#endif /* CONFIG_FUNCTION_TRACER */ 134#endif /* CONFIG_FUNCTION_TRACER */
134 135
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 659732eba07c..288541f977fb 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -101,6 +101,206 @@ static inline void warn_failed_init_tracer(struct tracer *trace, int init_ret)
101 101
102#ifdef CONFIG_DYNAMIC_FTRACE 102#ifdef CONFIG_DYNAMIC_FTRACE
103 103
104static int trace_selftest_test_probe1_cnt;
105static void trace_selftest_test_probe1_func(unsigned long ip,
106 unsigned long pip)
107{
108 trace_selftest_test_probe1_cnt++;
109}
110
111static int trace_selftest_test_probe2_cnt;
112static void trace_selftest_test_probe2_func(unsigned long ip,
113 unsigned long pip)
114{
115 trace_selftest_test_probe2_cnt++;
116}
117
118static int trace_selftest_test_probe3_cnt;
119static void trace_selftest_test_probe3_func(unsigned long ip,
120 unsigned long pip)
121{
122 trace_selftest_test_probe3_cnt++;
123}
124
125static int trace_selftest_test_global_cnt;
126static void trace_selftest_test_global_func(unsigned long ip,
127 unsigned long pip)
128{
129 trace_selftest_test_global_cnt++;
130}
131
132static int trace_selftest_test_dyn_cnt;
133static void trace_selftest_test_dyn_func(unsigned long ip,
134 unsigned long pip)
135{
136 trace_selftest_test_dyn_cnt++;
137}
138
139static struct ftrace_ops test_probe1 = {
140 .func = trace_selftest_test_probe1_func,
141};
142
143static struct ftrace_ops test_probe2 = {
144 .func = trace_selftest_test_probe2_func,
145};
146
147static struct ftrace_ops test_probe3 = {
148 .func = trace_selftest_test_probe3_func,
149};
150
151static struct ftrace_ops test_global = {
152 .func = trace_selftest_test_global_func,
153 .flags = FTRACE_OPS_FL_GLOBAL,
154};
155
156static void print_counts(void)
157{
158 printk("(%d %d %d %d %d) ",
159 trace_selftest_test_probe1_cnt,
160 trace_selftest_test_probe2_cnt,
161 trace_selftest_test_probe3_cnt,
162 trace_selftest_test_global_cnt,
163 trace_selftest_test_dyn_cnt);
164}
165
166static void reset_counts(void)
167{
168 trace_selftest_test_probe1_cnt = 0;
169 trace_selftest_test_probe2_cnt = 0;
170 trace_selftest_test_probe3_cnt = 0;
171 trace_selftest_test_global_cnt = 0;
172 trace_selftest_test_dyn_cnt = 0;
173}
174
175static int trace_selftest_ops(int cnt)
176{
177 int save_ftrace_enabled = ftrace_enabled;
178 struct ftrace_ops *dyn_ops;
179 char *func1_name;
180 char *func2_name;
181 int len1;
182 int len2;
183 int ret = -1;
184
185 printk(KERN_CONT "PASSED\n");
186 pr_info("Testing dynamic ftrace ops #%d: ", cnt);
187
188 ftrace_enabled = 1;
189 reset_counts();
190
191 /* Handle PPC64 '.' name */
192 func1_name = "*" __stringify(DYN_FTRACE_TEST_NAME);
193 func2_name = "*" __stringify(DYN_FTRACE_TEST_NAME2);
194 len1 = strlen(func1_name);
195 len2 = strlen(func2_name);
196
197 /*
198 * Probe 1 will trace function 1.
199 * Probe 2 will trace function 2.
200 * Probe 3 will trace functions 1 and 2.
201 */
202 ftrace_set_filter(&test_probe1, func1_name, len1, 1);
203 ftrace_set_filter(&test_probe2, func2_name, len2, 1);
204 ftrace_set_filter(&test_probe3, func1_name, len1, 1);
205 ftrace_set_filter(&test_probe3, func2_name, len2, 0);
206
207 register_ftrace_function(&test_probe1);
208 register_ftrace_function(&test_probe2);
209 register_ftrace_function(&test_probe3);
210 register_ftrace_function(&test_global);
211
212 DYN_FTRACE_TEST_NAME();
213
214 print_counts();
215
216 if (trace_selftest_test_probe1_cnt != 1)
217 goto out;
218 if (trace_selftest_test_probe2_cnt != 0)
219 goto out;
220 if (trace_selftest_test_probe3_cnt != 1)
221 goto out;
222 if (trace_selftest_test_global_cnt == 0)
223 goto out;
224
225 DYN_FTRACE_TEST_NAME2();
226
227 print_counts();
228
229 if (trace_selftest_test_probe1_cnt != 1)
230 goto out;
231 if (trace_selftest_test_probe2_cnt != 1)
232 goto out;
233 if (trace_selftest_test_probe3_cnt != 2)
234 goto out;
235
236 /* Add a dynamic probe */
237 dyn_ops = kzalloc(sizeof(*dyn_ops), GFP_KERNEL);
238 if (!dyn_ops) {
239 printk("MEMORY ERROR ");
240 goto out;
241 }
242
243 dyn_ops->func = trace_selftest_test_dyn_func;
244
245 register_ftrace_function(dyn_ops);
246
247 trace_selftest_test_global_cnt = 0;
248
249 DYN_FTRACE_TEST_NAME();
250
251 print_counts();
252
253 if (trace_selftest_test_probe1_cnt != 2)
254 goto out_free;
255 if (trace_selftest_test_probe2_cnt != 1)
256 goto out_free;
257 if (trace_selftest_test_probe3_cnt != 3)
258 goto out_free;
259 if (trace_selftest_test_global_cnt == 0)
260 goto out;
261 if (trace_selftest_test_dyn_cnt == 0)
262 goto out_free;
263
264 DYN_FTRACE_TEST_NAME2();
265
266 print_counts();
267
268 if (trace_selftest_test_probe1_cnt != 2)
269 goto out_free;
270 if (trace_selftest_test_probe2_cnt != 2)
271 goto out_free;
272 if (trace_selftest_test_probe3_cnt != 4)
273 goto out_free;
274
275 ret = 0;
276 out_free:
277 unregister_ftrace_function(dyn_ops);
278 kfree(dyn_ops);
279
280 out:
281 /* Purposely unregister in the same order */
282 unregister_ftrace_function(&test_probe1);
283 unregister_ftrace_function(&test_probe2);
284 unregister_ftrace_function(&test_probe3);
285 unregister_ftrace_function(&test_global);
286
287 /* Make sure everything is off */
288 reset_counts();
289 DYN_FTRACE_TEST_NAME();
290 DYN_FTRACE_TEST_NAME();
291
292 if (trace_selftest_test_probe1_cnt ||
293 trace_selftest_test_probe2_cnt ||
294 trace_selftest_test_probe3_cnt ||
295 trace_selftest_test_global_cnt ||
296 trace_selftest_test_dyn_cnt)
297 ret = -1;
298
299 ftrace_enabled = save_ftrace_enabled;
300
301 return ret;
302}
303
104/* Test dynamic code modification and ftrace filters */ 304/* Test dynamic code modification and ftrace filters */
105int trace_selftest_startup_dynamic_tracing(struct tracer *trace, 305int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
106 struct trace_array *tr, 306 struct trace_array *tr,
@@ -131,7 +331,7 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
131 func_name = "*" __stringify(DYN_FTRACE_TEST_NAME); 331 func_name = "*" __stringify(DYN_FTRACE_TEST_NAME);
132 332
133 /* filter only on our function */ 333 /* filter only on our function */
134 ftrace_set_filter(func_name, strlen(func_name), 1); 334 ftrace_set_global_filter(func_name, strlen(func_name), 1);
135 335
136 /* enable tracing */ 336 /* enable tracing */
137 ret = tracer_init(trace, tr); 337 ret = tracer_init(trace, tr);
@@ -166,22 +366,30 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
166 366
167 /* check the trace buffer */ 367 /* check the trace buffer */
168 ret = trace_test_buffer(tr, &count); 368 ret = trace_test_buffer(tr, &count);
169 trace->reset(tr);
170 tracing_start(); 369 tracing_start();
171 370
172 /* we should only have one item */ 371 /* we should only have one item */
173 if (!ret && count != 1) { 372 if (!ret && count != 1) {
373 trace->reset(tr);
174 printk(KERN_CONT ".. filter failed count=%ld ..", count); 374 printk(KERN_CONT ".. filter failed count=%ld ..", count);
175 ret = -1; 375 ret = -1;
176 goto out; 376 goto out;
177 } 377 }
178 378
379 /* Test the ops with global tracing running */
380 ret = trace_selftest_ops(1);
381 trace->reset(tr);
382
179 out: 383 out:
180 ftrace_enabled = save_ftrace_enabled; 384 ftrace_enabled = save_ftrace_enabled;
181 tracer_enabled = save_tracer_enabled; 385 tracer_enabled = save_tracer_enabled;
182 386
183 /* Enable tracing on all functions again */ 387 /* Enable tracing on all functions again */
184 ftrace_set_filter(NULL, 0, 1); 388 ftrace_set_global_filter(NULL, 0, 1);
389
390 /* Test the ops with global tracing off */
391 if (!ret)
392 ret = trace_selftest_ops(2);
185 393
186 return ret; 394 return ret;
187} 395}
diff --git a/kernel/trace/trace_selftest_dynamic.c b/kernel/trace/trace_selftest_dynamic.c
index 54dd77cce5bf..b4c475a0a48b 100644
--- a/kernel/trace/trace_selftest_dynamic.c
+++ b/kernel/trace/trace_selftest_dynamic.c
@@ -5,3 +5,9 @@ int DYN_FTRACE_TEST_NAME(void)
5 /* used to call mcount */ 5 /* used to call mcount */
6 return 0; 6 return 0;
7} 7}
8
9int DYN_FTRACE_TEST_NAME2(void)
10{
11 /* used to call mcount */
12 return 0;
13}
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c
index 4c5dead0c239..b0b53b8e4c25 100644
--- a/kernel/trace/trace_stack.c
+++ b/kernel/trace/trace_stack.c
@@ -133,6 +133,7 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip)
133static struct ftrace_ops trace_ops __read_mostly = 133static struct ftrace_ops trace_ops __read_mostly =
134{ 134{
135 .func = stack_trace_call, 135 .func = stack_trace_call,
136 .flags = FTRACE_OPS_FL_GLOBAL,
136}; 137};
137 138
138static ssize_t 139static ssize_t
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index 68187af4889e..b219f1449c54 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -251,9 +251,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
251{ 251{
252 WARN_ON(strcmp((*entry)->name, elem->name) != 0); 252 WARN_ON(strcmp((*entry)->name, elem->name) != 0);
253 253
254 if (elem->regfunc && !elem->state && active) 254 if (elem->regfunc && !jump_label_enabled(&elem->key) && active)
255 elem->regfunc(); 255 elem->regfunc();
256 else if (elem->unregfunc && elem->state && !active) 256 else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active)
257 elem->unregfunc(); 257 elem->unregfunc();
258 258
259 /* 259 /*
@@ -264,13 +264,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
264 * is used. 264 * is used.
265 */ 265 */
266 rcu_assign_pointer(elem->funcs, (*entry)->funcs); 266 rcu_assign_pointer(elem->funcs, (*entry)->funcs);
267 if (!elem->state && active) { 267 if (active && !jump_label_enabled(&elem->key))
268 jump_label_enable(&elem->state); 268 jump_label_inc(&elem->key);
269 elem->state = active; 269 else if (!active && jump_label_enabled(&elem->key))
270 } else if (elem->state && !active) { 270 jump_label_dec(&elem->key);
271 jump_label_disable(&elem->state);
272 elem->state = active;
273 }
274} 271}
275 272
276/* 273/*
@@ -281,13 +278,11 @@ static void set_tracepoint(struct tracepoint_entry **entry,
281 */ 278 */
282static void disable_tracepoint(struct tracepoint *elem) 279static void disable_tracepoint(struct tracepoint *elem)
283{ 280{
284 if (elem->unregfunc && elem->state) 281 if (elem->unregfunc && jump_label_enabled(&elem->key))
285 elem->unregfunc(); 282 elem->unregfunc();
286 283
287 if (elem->state) { 284 if (jump_label_enabled(&elem->key))
288 jump_label_disable(&elem->state); 285 jump_label_dec(&elem->key);
289 elem->state = 0;
290 }
291 rcu_assign_pointer(elem->funcs, NULL); 286 rcu_assign_pointer(elem->funcs, NULL);
292} 287}
293 288
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index d5f925abe4d2..6165622c3e29 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -244,14 +244,19 @@ endif
244 244
245ifdef CONFIG_FTRACE_MCOUNT_RECORD 245ifdef CONFIG_FTRACE_MCOUNT_RECORD
246ifdef BUILD_C_RECORDMCOUNT 246ifdef BUILD_C_RECORDMCOUNT
247ifeq ("$(origin RECORDMCOUNT_WARN)", "command line")
248 RECORDMCOUNT_FLAGS = -w
249endif
247# Due to recursion, we must skip empty.o. 250# Due to recursion, we must skip empty.o.
248# The empty.o file is created in the make process in order to determine 251# The empty.o file is created in the make process in order to determine
249# the target endianness and word size. It is made before all other C 252# the target endianness and word size. It is made before all other C
250# files, including recordmcount. 253# files, including recordmcount.
251sub_cmd_record_mcount = \ 254sub_cmd_record_mcount = \
252 if [ $(@) != "scripts/mod/empty.o" ]; then \ 255 if [ $(@) != "scripts/mod/empty.o" ]; then \
253 $(objtree)/scripts/recordmcount "$(@)"; \ 256 $(objtree)/scripts/recordmcount $(RECORDMCOUNT_FLAGS) "$(@)"; \
254 fi; 257 fi;
258recordmcount_source := $(srctree)/scripts/recordmcount.c \
259 $(srctree)/scripts/recordmcount.h
255else 260else
256sub_cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \ 261sub_cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \
257 "$(if $(CONFIG_CPU_BIG_ENDIAN),big,little)" \ 262 "$(if $(CONFIG_CPU_BIG_ENDIAN),big,little)" \
@@ -259,6 +264,7 @@ sub_cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH
259 "$(OBJDUMP)" "$(OBJCOPY)" "$(CC) $(KBUILD_CFLAGS)" \ 264 "$(OBJDUMP)" "$(OBJCOPY)" "$(CC) $(KBUILD_CFLAGS)" \
260 "$(LD)" "$(NM)" "$(RM)" "$(MV)" \ 265 "$(LD)" "$(NM)" "$(RM)" "$(MV)" \
261 "$(if $(part-of-module),1,0)" "$(@)"; 266 "$(if $(part-of-module),1,0)" "$(@)";
267recordmcount_source := $(srctree)/scripts/recordmcount.pl
262endif 268endif
263cmd_record_mcount = \ 269cmd_record_mcount = \
264 if [ "$(findstring -pg,$(_c_flags))" = "-pg" ]; then \ 270 if [ "$(findstring -pg,$(_c_flags))" = "-pg" ]; then \
@@ -279,13 +285,13 @@ define rule_cc_o_c
279endef 285endef
280 286
281# Built-in and composite module parts 287# Built-in and composite module parts
282$(obj)/%.o: $(src)/%.c FORCE 288$(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE
283 $(call cmd,force_checksrc) 289 $(call cmd,force_checksrc)
284 $(call if_changed_rule,cc_o_c) 290 $(call if_changed_rule,cc_o_c)
285 291
286# Single-part modules are special since we need to mark them in $(MODVERDIR) 292# Single-part modules are special since we need to mark them in $(MODVERDIR)
287 293
288$(single-used-m): $(obj)/%.o: $(src)/%.c FORCE 294$(single-used-m): $(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE
289 $(call cmd,force_checksrc) 295 $(call cmd,force_checksrc)
290 $(call if_changed_rule,cc_o_c) 296 $(call if_changed_rule,cc_o_c)
291 @{ echo $(@:.o=.ko); echo $@; } > $(MODVERDIR)/$(@F:.o=.mod) 297 @{ echo $(@:.o=.ko); echo $@; } > $(MODVERDIR)/$(@F:.o=.mod)
diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c
index f9f6f52db772..ee52cb8e17ad 100644
--- a/scripts/recordmcount.c
+++ b/scripts/recordmcount.c
@@ -24,6 +24,7 @@
24#include <sys/types.h> 24#include <sys/types.h>
25#include <sys/mman.h> 25#include <sys/mman.h>
26#include <sys/stat.h> 26#include <sys/stat.h>
27#include <getopt.h>
27#include <elf.h> 28#include <elf.h>
28#include <fcntl.h> 29#include <fcntl.h>
29#include <setjmp.h> 30#include <setjmp.h>
@@ -39,6 +40,7 @@ static char gpfx; /* prefix for global symbol name (sometimes '_') */
39static struct stat sb; /* Remember .st_size, etc. */ 40static struct stat sb; /* Remember .st_size, etc. */
40static jmp_buf jmpenv; /* setjmp/longjmp per-file error escape */ 41static jmp_buf jmpenv; /* setjmp/longjmp per-file error escape */
41static const char *altmcount; /* alternate mcount symbol name */ 42static const char *altmcount; /* alternate mcount symbol name */
43static int warn_on_notrace_sect; /* warn when section has mcount not being recorded */
42 44
43/* setjmp() return values */ 45/* setjmp() return values */
44enum { 46enum {
@@ -78,7 +80,7 @@ static off_t
78ulseek(int const fd, off_t const offset, int const whence) 80ulseek(int const fd, off_t const offset, int const whence)
79{ 81{
80 off_t const w = lseek(fd, offset, whence); 82 off_t const w = lseek(fd, offset, whence);
81 if ((off_t)-1 == w) { 83 if (w == (off_t)-1) {
82 perror("lseek"); 84 perror("lseek");
83 fail_file(); 85 fail_file();
84 } 86 }
@@ -111,13 +113,41 @@ static void *
111umalloc(size_t size) 113umalloc(size_t size)
112{ 114{
113 void *const addr = malloc(size); 115 void *const addr = malloc(size);
114 if (0 == addr) { 116 if (addr == 0) {
115 fprintf(stderr, "malloc failed: %zu bytes\n", size); 117 fprintf(stderr, "malloc failed: %zu bytes\n", size);
116 fail_file(); 118 fail_file();
117 } 119 }
118 return addr; 120 return addr;
119} 121}
120 122
123static unsigned char ideal_nop5_x86_64[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 };
124static unsigned char ideal_nop5_x86_32[5] = { 0x3e, 0x8d, 0x74, 0x26, 0x00 };
125static unsigned char *ideal_nop;
126
127static char rel_type_nop;
128
129static int (*make_nop)(void *map, size_t const offset);
130
131static int make_nop_x86(void *map, size_t const offset)
132{
133 uint32_t *ptr;
134 unsigned char *op;
135
136 /* Confirm we have 0xe8 0x0 0x0 0x0 0x0 */
137 ptr = map + offset;
138 if (*ptr != 0)
139 return -1;
140
141 op = map + offset - 1;
142 if (*op != 0xe8)
143 return -1;
144
145 /* convert to nop */
146 ulseek(fd_map, offset - 1, SEEK_SET);
147 uwrite(fd_map, ideal_nop, 5);
148 return 0;
149}
150
121/* 151/*
122 * Get the whole file as a programming convenience in order to avoid 152 * Get the whole file as a programming convenience in order to avoid
123 * malloc+lseek+read+free of many pieces. If successful, then mmap 153 * malloc+lseek+read+free of many pieces. If successful, then mmap
@@ -136,7 +166,7 @@ static void *mmap_file(char const *fname)
136 void *addr; 166 void *addr;
137 167
138 fd_map = open(fname, O_RDWR); 168 fd_map = open(fname, O_RDWR);
139 if (0 > fd_map || 0 > fstat(fd_map, &sb)) { 169 if (fd_map < 0 || fstat(fd_map, &sb) < 0) {
140 perror(fname); 170 perror(fname);
141 fail_file(); 171 fail_file();
142 } 172 }
@@ -147,7 +177,7 @@ static void *mmap_file(char const *fname)
147 addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE, 177 addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE,
148 fd_map, 0); 178 fd_map, 0);
149 mmap_failed = 0; 179 mmap_failed = 0;
150 if (MAP_FAILED == addr) { 180 if (addr == MAP_FAILED) {
151 mmap_failed = 1; 181 mmap_failed = 1;
152 addr = umalloc(sb.st_size); 182 addr = umalloc(sb.st_size);
153 uread(fd_map, addr, sb.st_size); 183 uread(fd_map, addr, sb.st_size);
@@ -206,12 +236,13 @@ static uint32_t (*w2)(uint16_t);
206static int 236static int
207is_mcounted_section_name(char const *const txtname) 237is_mcounted_section_name(char const *const txtname)
208{ 238{
209 return 0 == strcmp(".text", txtname) || 239 return strcmp(".text", txtname) == 0 ||
210 0 == strcmp(".ref.text", txtname) || 240 strcmp(".ref.text", txtname) == 0 ||
211 0 == strcmp(".sched.text", txtname) || 241 strcmp(".sched.text", txtname) == 0 ||
212 0 == strcmp(".spinlock.text", txtname) || 242 strcmp(".spinlock.text", txtname) == 0 ||
213 0 == strcmp(".irqentry.text", txtname) || 243 strcmp(".irqentry.text", txtname) == 0 ||
214 0 == strcmp(".text.unlikely", txtname); 244 strcmp(".kprobes.text", txtname) == 0 ||
245 strcmp(".text.unlikely", txtname) == 0;
215} 246}
216 247
217/* 32 bit and 64 bit are very similar */ 248/* 32 bit and 64 bit are very similar */
@@ -264,43 +295,48 @@ do_file(char const *const fname)
264 w8 = w8nat; 295 w8 = w8nat;
265 switch (ehdr->e_ident[EI_DATA]) { 296 switch (ehdr->e_ident[EI_DATA]) {
266 static unsigned int const endian = 1; 297 static unsigned int const endian = 1;
267 default: { 298 default:
268 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", 299 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n",
269 ehdr->e_ident[EI_DATA], fname); 300 ehdr->e_ident[EI_DATA], fname);
270 fail_file(); 301 fail_file();
271 } break; 302 break;
272 case ELFDATA2LSB: { 303 case ELFDATA2LSB:
273 if (1 != *(unsigned char const *)&endian) { 304 if (*(unsigned char const *)&endian != 1) {
274 /* main() is big endian, file.o is little endian. */ 305 /* main() is big endian, file.o is little endian. */
275 w = w4rev; 306 w = w4rev;
276 w2 = w2rev; 307 w2 = w2rev;
277 w8 = w8rev; 308 w8 = w8rev;
278 } 309 }
279 } break; 310 break;
280 case ELFDATA2MSB: { 311 case ELFDATA2MSB:
281 if (0 != *(unsigned char const *)&endian) { 312 if (*(unsigned char const *)&endian != 0) {
282 /* main() is little endian, file.o is big endian. */ 313 /* main() is little endian, file.o is big endian. */
283 w = w4rev; 314 w = w4rev;
284 w2 = w2rev; 315 w2 = w2rev;
285 w8 = w8rev; 316 w8 = w8rev;
286 } 317 }
287 } break; 318 break;
288 } /* end switch */ 319 } /* end switch */
289 if (0 != memcmp(ELFMAG, ehdr->e_ident, SELFMAG) 320 if (memcmp(ELFMAG, ehdr->e_ident, SELFMAG) != 0
290 || ET_REL != w2(ehdr->e_type) 321 || w2(ehdr->e_type) != ET_REL
291 || EV_CURRENT != ehdr->e_ident[EI_VERSION]) { 322 || ehdr->e_ident[EI_VERSION] != EV_CURRENT) {
292 fprintf(stderr, "unrecognized ET_REL file %s\n", fname); 323 fprintf(stderr, "unrecognized ET_REL file %s\n", fname);
293 fail_file(); 324 fail_file();
294 } 325 }
295 326
296 gpfx = 0; 327 gpfx = 0;
297 switch (w2(ehdr->e_machine)) { 328 switch (w2(ehdr->e_machine)) {
298 default: { 329 default:
299 fprintf(stderr, "unrecognized e_machine %d %s\n", 330 fprintf(stderr, "unrecognized e_machine %d %s\n",
300 w2(ehdr->e_machine), fname); 331 w2(ehdr->e_machine), fname);
301 fail_file(); 332 fail_file();
302 } break; 333 break;
303 case EM_386: reltype = R_386_32; break; 334 case EM_386:
335 reltype = R_386_32;
336 make_nop = make_nop_x86;
337 ideal_nop = ideal_nop5_x86_32;
338 mcount_adjust_32 = -1;
339 break;
304 case EM_ARM: reltype = R_ARM_ABS32; 340 case EM_ARM: reltype = R_ARM_ABS32;
305 altmcount = "__gnu_mcount_nc"; 341 altmcount = "__gnu_mcount_nc";
306 break; 342 break;
@@ -311,67 +347,91 @@ do_file(char const *const fname)
311 case EM_S390: /* reltype: e_class */ gpfx = '_'; break; 347 case EM_S390: /* reltype: e_class */ gpfx = '_'; break;
312 case EM_SH: reltype = R_SH_DIR32; break; 348 case EM_SH: reltype = R_SH_DIR32; break;
313 case EM_SPARCV9: reltype = R_SPARC_64; gpfx = '_'; break; 349 case EM_SPARCV9: reltype = R_SPARC_64; gpfx = '_'; break;
314 case EM_X86_64: reltype = R_X86_64_64; break; 350 case EM_X86_64:
351 make_nop = make_nop_x86;
352 ideal_nop = ideal_nop5_x86_64;
353 reltype = R_X86_64_64;
354 mcount_adjust_64 = -1;
355 break;
315 } /* end switch */ 356 } /* end switch */
316 357
317 switch (ehdr->e_ident[EI_CLASS]) { 358 switch (ehdr->e_ident[EI_CLASS]) {
318 default: { 359 default:
319 fprintf(stderr, "unrecognized ELF class %d %s\n", 360 fprintf(stderr, "unrecognized ELF class %d %s\n",
320 ehdr->e_ident[EI_CLASS], fname); 361 ehdr->e_ident[EI_CLASS], fname);
321 fail_file(); 362 fail_file();
322 } break; 363 break;
323 case ELFCLASS32: { 364 case ELFCLASS32:
324 if (sizeof(Elf32_Ehdr) != w2(ehdr->e_ehsize) 365 if (w2(ehdr->e_ehsize) != sizeof(Elf32_Ehdr)
325 || sizeof(Elf32_Shdr) != w2(ehdr->e_shentsize)) { 366 || w2(ehdr->e_shentsize) != sizeof(Elf32_Shdr)) {
326 fprintf(stderr, 367 fprintf(stderr,
327 "unrecognized ET_REL file: %s\n", fname); 368 "unrecognized ET_REL file: %s\n", fname);
328 fail_file(); 369 fail_file();
329 } 370 }
330 if (EM_S390 == w2(ehdr->e_machine)) 371 if (w2(ehdr->e_machine) == EM_S390) {
331 reltype = R_390_32; 372 reltype = R_390_32;
332 if (EM_MIPS == w2(ehdr->e_machine)) { 373 mcount_adjust_32 = -4;
374 }
375 if (w2(ehdr->e_machine) == EM_MIPS) {
333 reltype = R_MIPS_32; 376 reltype = R_MIPS_32;
334 is_fake_mcount32 = MIPS32_is_fake_mcount; 377 is_fake_mcount32 = MIPS32_is_fake_mcount;
335 } 378 }
336 do32(ehdr, fname, reltype); 379 do32(ehdr, fname, reltype);
337 } break; 380 break;
338 case ELFCLASS64: { 381 case ELFCLASS64: {
339 Elf64_Ehdr *const ghdr = (Elf64_Ehdr *)ehdr; 382 Elf64_Ehdr *const ghdr = (Elf64_Ehdr *)ehdr;
340 if (sizeof(Elf64_Ehdr) != w2(ghdr->e_ehsize) 383 if (w2(ghdr->e_ehsize) != sizeof(Elf64_Ehdr)
341 || sizeof(Elf64_Shdr) != w2(ghdr->e_shentsize)) { 384 || w2(ghdr->e_shentsize) != sizeof(Elf64_Shdr)) {
342 fprintf(stderr, 385 fprintf(stderr,
343 "unrecognized ET_REL file: %s\n", fname); 386 "unrecognized ET_REL file: %s\n", fname);
344 fail_file(); 387 fail_file();
345 } 388 }
346 if (EM_S390 == w2(ghdr->e_machine)) 389 if (w2(ghdr->e_machine) == EM_S390) {
347 reltype = R_390_64; 390 reltype = R_390_64;
348 if (EM_MIPS == w2(ghdr->e_machine)) { 391 mcount_adjust_64 = -8;
392 }
393 if (w2(ghdr->e_machine) == EM_MIPS) {
349 reltype = R_MIPS_64; 394 reltype = R_MIPS_64;
350 Elf64_r_sym = MIPS64_r_sym; 395 Elf64_r_sym = MIPS64_r_sym;
351 Elf64_r_info = MIPS64_r_info; 396 Elf64_r_info = MIPS64_r_info;
352 is_fake_mcount64 = MIPS64_is_fake_mcount; 397 is_fake_mcount64 = MIPS64_is_fake_mcount;
353 } 398 }
354 do64(ghdr, fname, reltype); 399 do64(ghdr, fname, reltype);
355 } break; 400 break;
401 }
356 } /* end switch */ 402 } /* end switch */
357 403
358 cleanup(); 404 cleanup();
359} 405}
360 406
361int 407int
362main(int argc, char const *argv[]) 408main(int argc, char *argv[])
363{ 409{
364 const char ftrace[] = "/ftrace.o"; 410 const char ftrace[] = "/ftrace.o";
365 int ftrace_size = sizeof(ftrace) - 1; 411 int ftrace_size = sizeof(ftrace) - 1;
366 int n_error = 0; /* gcc-4.3.0 false positive complaint */ 412 int n_error = 0; /* gcc-4.3.0 false positive complaint */
413 int c;
414 int i;
415
416 while ((c = getopt(argc, argv, "w")) >= 0) {
417 switch (c) {
418 case 'w':
419 warn_on_notrace_sect = 1;
420 break;
421 default:
422 fprintf(stderr, "usage: recordmcount [-w] file.o...\n");
423 return 0;
424 }
425 }
367 426
368 if (argc <= 1) { 427 if ((argc - optind) < 1) {
369 fprintf(stderr, "usage: recordmcount file.o...\n"); 428 fprintf(stderr, "usage: recordmcount [-w] file.o...\n");
370 return 0; 429 return 0;
371 } 430 }
372 431
373 /* Process each file in turn, allowing deep failure. */ 432 /* Process each file in turn, allowing deep failure. */
374 for (--argc, ++argv; 0 < argc; --argc, ++argv) { 433 for (i = optind; i < argc; i++) {
434 char *file = argv[i];
375 int const sjval = setjmp(jmpenv); 435 int const sjval = setjmp(jmpenv);
376 int len; 436 int len;
377 437
@@ -380,29 +440,29 @@ main(int argc, char const *argv[])
380 * function but does not call it. Since ftrace.o should 440 * function but does not call it. Since ftrace.o should
381 * not be traced anyway, we just skip it. 441 * not be traced anyway, we just skip it.
382 */ 442 */
383 len = strlen(argv[0]); 443 len = strlen(file);
384 if (len >= ftrace_size && 444 if (len >= ftrace_size &&
385 strcmp(argv[0] + (len - ftrace_size), ftrace) == 0) 445 strcmp(file + (len - ftrace_size), ftrace) == 0)
386 continue; 446 continue;
387 447
388 switch (sjval) { 448 switch (sjval) {
389 default: { 449 default:
390 fprintf(stderr, "internal error: %s\n", argv[0]); 450 fprintf(stderr, "internal error: %s\n", file);
391 exit(1); 451 exit(1);
392 } break; 452 break;
393 case SJ_SETJMP: { /* normal sequence */ 453 case SJ_SETJMP: /* normal sequence */
394 /* Avoid problems if early cleanup() */ 454 /* Avoid problems if early cleanup() */
395 fd_map = -1; 455 fd_map = -1;
396 ehdr_curr = NULL; 456 ehdr_curr = NULL;
397 mmap_failed = 1; 457 mmap_failed = 1;
398 do_file(argv[0]); 458 do_file(file);
399 } break; 459 break;
400 case SJ_FAIL: { /* error in do_file or below */ 460 case SJ_FAIL: /* error in do_file or below */
401 ++n_error; 461 ++n_error;
402 } break; 462 break;
403 case SJ_SUCCEED: { /* premature success */ 463 case SJ_SUCCEED: /* premature success */
404 /* do nothing */ 464 /* do nothing */
405 } break; 465 break;
406 } /* end switch */ 466 } /* end switch */
407 } 467 }
408 return !!n_error; 468 return !!n_error;
diff --git a/scripts/recordmcount.h b/scripts/recordmcount.h
index baf187bee983..4be60364a405 100644
--- a/scripts/recordmcount.h
+++ b/scripts/recordmcount.h
@@ -22,11 +22,15 @@
22#undef is_fake_mcount 22#undef is_fake_mcount
23#undef fn_is_fake_mcount 23#undef fn_is_fake_mcount
24#undef MIPS_is_fake_mcount 24#undef MIPS_is_fake_mcount
25#undef mcount_adjust
25#undef sift_rel_mcount 26#undef sift_rel_mcount
27#undef nop_mcount
26#undef find_secsym_ndx 28#undef find_secsym_ndx
27#undef __has_rel_mcount 29#undef __has_rel_mcount
28#undef has_rel_mcount 30#undef has_rel_mcount
29#undef tot_relsize 31#undef tot_relsize
32#undef get_mcountsym
33#undef get_sym_str_and_relp
30#undef do_func 34#undef do_func
31#undef Elf_Addr 35#undef Elf_Addr
32#undef Elf_Ehdr 36#undef Elf_Ehdr
@@ -49,14 +53,18 @@
49#ifdef RECORD_MCOUNT_64 53#ifdef RECORD_MCOUNT_64
50# define append_func append64 54# define append_func append64
51# define sift_rel_mcount sift64_rel_mcount 55# define sift_rel_mcount sift64_rel_mcount
56# define nop_mcount nop_mcount_64
52# define find_secsym_ndx find64_secsym_ndx 57# define find_secsym_ndx find64_secsym_ndx
53# define __has_rel_mcount __has64_rel_mcount 58# define __has_rel_mcount __has64_rel_mcount
54# define has_rel_mcount has64_rel_mcount 59# define has_rel_mcount has64_rel_mcount
55# define tot_relsize tot64_relsize 60# define tot_relsize tot64_relsize
61# define get_sym_str_and_relp get_sym_str_and_relp_64
56# define do_func do64 62# define do_func do64
63# define get_mcountsym get_mcountsym_64
57# define is_fake_mcount is_fake_mcount64 64# define is_fake_mcount is_fake_mcount64
58# define fn_is_fake_mcount fn_is_fake_mcount64 65# define fn_is_fake_mcount fn_is_fake_mcount64
59# define MIPS_is_fake_mcount MIPS64_is_fake_mcount 66# define MIPS_is_fake_mcount MIPS64_is_fake_mcount
67# define mcount_adjust mcount_adjust_64
60# define Elf_Addr Elf64_Addr 68# define Elf_Addr Elf64_Addr
61# define Elf_Ehdr Elf64_Ehdr 69# define Elf_Ehdr Elf64_Ehdr
62# define Elf_Shdr Elf64_Shdr 70# define Elf_Shdr Elf64_Shdr
@@ -77,14 +85,18 @@
77#else 85#else
78# define append_func append32 86# define append_func append32
79# define sift_rel_mcount sift32_rel_mcount 87# define sift_rel_mcount sift32_rel_mcount
88# define nop_mcount nop_mcount_32
80# define find_secsym_ndx find32_secsym_ndx 89# define find_secsym_ndx find32_secsym_ndx
81# define __has_rel_mcount __has32_rel_mcount 90# define __has_rel_mcount __has32_rel_mcount
82# define has_rel_mcount has32_rel_mcount 91# define has_rel_mcount has32_rel_mcount
83# define tot_relsize tot32_relsize 92# define tot_relsize tot32_relsize
93# define get_sym_str_and_relp get_sym_str_and_relp_32
84# define do_func do32 94# define do_func do32
95# define get_mcountsym get_mcountsym_32
85# define is_fake_mcount is_fake_mcount32 96# define is_fake_mcount is_fake_mcount32
86# define fn_is_fake_mcount fn_is_fake_mcount32 97# define fn_is_fake_mcount fn_is_fake_mcount32
87# define MIPS_is_fake_mcount MIPS32_is_fake_mcount 98# define MIPS_is_fake_mcount MIPS32_is_fake_mcount
99# define mcount_adjust mcount_adjust_32
88# define Elf_Addr Elf32_Addr 100# define Elf_Addr Elf32_Addr
89# define Elf_Ehdr Elf32_Ehdr 101# define Elf_Ehdr Elf32_Ehdr
90# define Elf_Shdr Elf32_Shdr 102# define Elf_Shdr Elf32_Shdr
@@ -123,6 +135,8 @@ static void fn_ELF_R_INFO(Elf_Rel *const rp, unsigned sym, unsigned type)
123} 135}
124static void (*Elf_r_info)(Elf_Rel *const rp, unsigned sym, unsigned type) = fn_ELF_R_INFO; 136static void (*Elf_r_info)(Elf_Rel *const rp, unsigned sym, unsigned type) = fn_ELF_R_INFO;
125 137
138static int mcount_adjust = 0;
139
126/* 140/*
127 * MIPS mcount long call has 2 _mcount symbols, only the position of the 1st 141 * MIPS mcount long call has 2 _mcount symbols, only the position of the 1st
128 * _mcount symbol is needed for dynamic function tracer, with it, to disable 142 * _mcount symbol is needed for dynamic function tracer, with it, to disable
@@ -234,6 +248,49 @@ static void append_func(Elf_Ehdr *const ehdr,
234 uwrite(fd_map, ehdr, sizeof(*ehdr)); 248 uwrite(fd_map, ehdr, sizeof(*ehdr));
235} 249}
236 250
251static unsigned get_mcountsym(Elf_Sym const *const sym0,
252 Elf_Rel const *relp,
253 char const *const str0)
254{
255 unsigned mcountsym = 0;
256
257 Elf_Sym const *const symp =
258 &sym0[Elf_r_sym(relp)];
259 char const *symname = &str0[w(symp->st_name)];
260 char const *mcount = gpfx == '_' ? "_mcount" : "mcount";
261
262 if (symname[0] == '.')
263 ++symname; /* ppc64 hack */
264 if (strcmp(mcount, symname) == 0 ||
265 (altmcount && strcmp(altmcount, symname) == 0))
266 mcountsym = Elf_r_sym(relp);
267
268 return mcountsym;
269}
270
271static void get_sym_str_and_relp(Elf_Shdr const *const relhdr,
272 Elf_Ehdr const *const ehdr,
273 Elf_Sym const **sym0,
274 char const **str0,
275 Elf_Rel const **relp)
276{
277 Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff)
278 + (void *)ehdr);
279 unsigned const symsec_sh_link = w(relhdr->sh_link);
280 Elf_Shdr const *const symsec = &shdr0[symsec_sh_link];
281 Elf_Shdr const *const strsec = &shdr0[w(symsec->sh_link)];
282 Elf_Rel const *const rel0 = (Elf_Rel const *)(_w(relhdr->sh_offset)
283 + (void *)ehdr);
284
285 *sym0 = (Elf_Sym const *)(_w(symsec->sh_offset)
286 + (void *)ehdr);
287
288 *str0 = (char const *)(_w(strsec->sh_offset)
289 + (void *)ehdr);
290
291 *relp = rel0;
292}
293
237/* 294/*
238 * Look at the relocations in order to find the calls to mcount. 295 * Look at the relocations in order to find the calls to mcount.
239 * Accumulate the section offsets that are found, and their relocation info, 296 * Accumulate the section offsets that are found, and their relocation info,
@@ -250,47 +307,27 @@ static uint_t *sift_rel_mcount(uint_t *mlocp,
250{ 307{
251 uint_t *const mloc0 = mlocp; 308 uint_t *const mloc0 = mlocp;
252 Elf_Rel *mrelp = *mrelpp; 309 Elf_Rel *mrelp = *mrelpp;
253 Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff) 310 Elf_Sym const *sym0;
254 + (void *)ehdr); 311 char const *str0;
255 unsigned const symsec_sh_link = w(relhdr->sh_link); 312 Elf_Rel const *relp;
256 Elf_Shdr const *const symsec = &shdr0[symsec_sh_link];
257 Elf_Sym const *const sym0 = (Elf_Sym const *)(_w(symsec->sh_offset)
258 + (void *)ehdr);
259
260 Elf_Shdr const *const strsec = &shdr0[w(symsec->sh_link)];
261 char const *const str0 = (char const *)(_w(strsec->sh_offset)
262 + (void *)ehdr);
263
264 Elf_Rel const *const rel0 = (Elf_Rel const *)(_w(relhdr->sh_offset)
265 + (void *)ehdr);
266 unsigned rel_entsize = _w(relhdr->sh_entsize); 313 unsigned rel_entsize = _w(relhdr->sh_entsize);
267 unsigned const nrel = _w(relhdr->sh_size) / rel_entsize; 314 unsigned const nrel = _w(relhdr->sh_size) / rel_entsize;
268 Elf_Rel const *relp = rel0;
269
270 unsigned mcountsym = 0; 315 unsigned mcountsym = 0;
271 unsigned t; 316 unsigned t;
272 317
318 get_sym_str_and_relp(relhdr, ehdr, &sym0, &str0, &relp);
319
273 for (t = nrel; t; --t) { 320 for (t = nrel; t; --t) {
274 if (!mcountsym) { 321 if (!mcountsym)
275 Elf_Sym const *const symp = 322 mcountsym = get_mcountsym(sym0, relp, str0);
276 &sym0[Elf_r_sym(relp)];
277 char const *symname = &str0[w(symp->st_name)];
278 char const *mcount = '_' == gpfx ? "_mcount" : "mcount";
279
280 if ('.' == symname[0])
281 ++symname; /* ppc64 hack */
282 if (0 == strcmp(mcount, symname) ||
283 (altmcount && 0 == strcmp(altmcount, symname)))
284 mcountsym = Elf_r_sym(relp);
285 }
286 323
287 if (mcountsym == Elf_r_sym(relp) && !is_fake_mcount(relp)) { 324 if (mcountsym == Elf_r_sym(relp) && !is_fake_mcount(relp)) {
288 uint_t const addend = _w(_w(relp->r_offset) - recval); 325 uint_t const addend =
289 326 _w(_w(relp->r_offset) - recval + mcount_adjust);
290 mrelp->r_offset = _w(offbase 327 mrelp->r_offset = _w(offbase
291 + ((void *)mlocp - (void *)mloc0)); 328 + ((void *)mlocp - (void *)mloc0));
292 Elf_r_info(mrelp, recsym, reltype); 329 Elf_r_info(mrelp, recsym, reltype);
293 if (sizeof(Elf_Rela) == rel_entsize) { 330 if (rel_entsize == sizeof(Elf_Rela)) {
294 ((Elf_Rela *)mrelp)->r_addend = addend; 331 ((Elf_Rela *)mrelp)->r_addend = addend;
295 *mlocp++ = 0; 332 *mlocp++ = 0;
296 } else 333 } else
@@ -304,6 +341,63 @@ static uint_t *sift_rel_mcount(uint_t *mlocp,
304 return mlocp; 341 return mlocp;
305} 342}
306 343
344/*
345 * Read the relocation table again, but this time its called on sections
346 * that are not going to be traced. The mcount calls here will be converted
347 * into nops.
348 */
349static void nop_mcount(Elf_Shdr const *const relhdr,
350 Elf_Ehdr const *const ehdr,
351 const char *const txtname)
352{
353 Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff)
354 + (void *)ehdr);
355 Elf_Sym const *sym0;
356 char const *str0;
357 Elf_Rel const *relp;
358 Elf_Shdr const *const shdr = &shdr0[w(relhdr->sh_info)];
359 unsigned rel_entsize = _w(relhdr->sh_entsize);
360 unsigned const nrel = _w(relhdr->sh_size) / rel_entsize;
361 unsigned mcountsym = 0;
362 unsigned t;
363 int once = 0;
364
365 get_sym_str_and_relp(relhdr, ehdr, &sym0, &str0, &relp);
366
367 for (t = nrel; t; --t) {
368 int ret = -1;
369
370 if (!mcountsym)
371 mcountsym = get_mcountsym(sym0, relp, str0);
372
373 if (mcountsym == Elf_r_sym(relp) && !is_fake_mcount(relp)) {
374 if (make_nop)
375 ret = make_nop((void *)ehdr, shdr->sh_offset + relp->r_offset);
376 if (warn_on_notrace_sect && !once) {
377 printf("Section %s has mcount callers being ignored\n",
378 txtname);
379 once = 1;
380 /* just warn? */
381 if (!make_nop)
382 return;
383 }
384 }
385
386 /*
387 * If we successfully removed the mcount, mark the relocation
388 * as a nop (don't do anything with it).
389 */
390 if (!ret) {
391 Elf_Rel rel;
392 rel = *(Elf_Rel *)relp;
393 Elf_r_info(&rel, Elf_r_sym(relp), rel_type_nop);
394 ulseek(fd_map, (void *)relp - (void *)ehdr, SEEK_SET);
395 uwrite(fd_map, &rel, sizeof(rel));
396 }
397 relp = (Elf_Rel const *)(rel_entsize + (void *)relp);
398 }
399}
400
307 401
308/* 402/*
309 * Find a symbol in the given section, to be used as the base for relocating 403 * Find a symbol in the given section, to be used as the base for relocating
@@ -354,13 +448,13 @@ __has_rel_mcount(Elf_Shdr const *const relhdr, /* is SHT_REL or SHT_RELA */
354 Elf_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)]; 448 Elf_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)];
355 char const *const txtname = &shstrtab[w(txthdr->sh_name)]; 449 char const *const txtname = &shstrtab[w(txthdr->sh_name)];
356 450
357 if (0 == strcmp("__mcount_loc", txtname)) { 451 if (strcmp("__mcount_loc", txtname) == 0) {
358 fprintf(stderr, "warning: __mcount_loc already exists: %s\n", 452 fprintf(stderr, "warning: __mcount_loc already exists: %s\n",
359 fname); 453 fname);
360 succeed_file(); 454 succeed_file();
361 } 455 }
362 if (SHT_PROGBITS != w(txthdr->sh_type) || 456 if (w(txthdr->sh_type) != SHT_PROGBITS ||
363 !is_mcounted_section_name(txtname)) 457 !(w(txthdr->sh_flags) & SHF_EXECINSTR))
364 return NULL; 458 return NULL;
365 return txtname; 459 return txtname;
366} 460}
@@ -370,7 +464,7 @@ static char const *has_rel_mcount(Elf_Shdr const *const relhdr,
370 char const *const shstrtab, 464 char const *const shstrtab,
371 char const *const fname) 465 char const *const fname)
372{ 466{
373 if (SHT_REL != w(relhdr->sh_type) && SHT_RELA != w(relhdr->sh_type)) 467 if (w(relhdr->sh_type) != SHT_REL && w(relhdr->sh_type) != SHT_RELA)
374 return NULL; 468 return NULL;
375 return __has_rel_mcount(relhdr, shdr0, shstrtab, fname); 469 return __has_rel_mcount(relhdr, shdr0, shstrtab, fname);
376} 470}
@@ -383,9 +477,11 @@ static unsigned tot_relsize(Elf_Shdr const *const shdr0,
383{ 477{
384 unsigned totrelsz = 0; 478 unsigned totrelsz = 0;
385 Elf_Shdr const *shdrp = shdr0; 479 Elf_Shdr const *shdrp = shdr0;
480 char const *txtname;
386 481
387 for (; nhdr; --nhdr, ++shdrp) { 482 for (; nhdr; --nhdr, ++shdrp) {
388 if (has_rel_mcount(shdrp, shdr0, shstrtab, fname)) 483 txtname = has_rel_mcount(shdrp, shdr0, shstrtab, fname);
484 if (txtname && is_mcounted_section_name(txtname))
389 totrelsz += _w(shdrp->sh_size); 485 totrelsz += _w(shdrp->sh_size);
390 } 486 }
391 return totrelsz; 487 return totrelsz;
@@ -421,7 +517,7 @@ do_func(Elf_Ehdr *const ehdr, char const *const fname, unsigned const reltype)
421 for (relhdr = shdr0, k = nhdr; k; --k, ++relhdr) { 517 for (relhdr = shdr0, k = nhdr; k; --k, ++relhdr) {
422 char const *const txtname = has_rel_mcount(relhdr, shdr0, 518 char const *const txtname = has_rel_mcount(relhdr, shdr0,
423 shstrtab, fname); 519 shstrtab, fname);
424 if (txtname) { 520 if (txtname && is_mcounted_section_name(txtname)) {
425 uint_t recval = 0; 521 uint_t recval = 0;
426 unsigned const recsym = find_secsym_ndx( 522 unsigned const recsym = find_secsym_ndx(
427 w(relhdr->sh_info), txtname, &recval, 523 w(relhdr->sh_info), txtname, &recval,
@@ -432,6 +528,12 @@ do_func(Elf_Ehdr *const ehdr, char const *const fname, unsigned const reltype)
432 mlocp = sift_rel_mcount(mlocp, 528 mlocp = sift_rel_mcount(mlocp,
433 (void *)mlocp - (void *)mloc0, &mrelp, 529 (void *)mlocp - (void *)mloc0, &mrelp,
434 relhdr, ehdr, recsym, recval, reltype); 530 relhdr, ehdr, recsym, recval, reltype);
531 } else if (txtname && (warn_on_notrace_sect || make_nop)) {
532 /*
533 * This section is ignored by ftrace, but still
534 * has mcount calls. Convert them to nops now.
535 */
536 nop_mcount(relhdr, ehdr, txtname);
435 } 537 }
436 } 538 }
437 if (mloc0 != mlocp) { 539 if (mloc0 != mlocp) {
diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl
index 4be0deea71ca..858966ab019c 100755
--- a/scripts/recordmcount.pl
+++ b/scripts/recordmcount.pl
@@ -134,6 +134,7 @@ my %text_sections = (
134 ".sched.text" => 1, 134 ".sched.text" => 1,
135 ".spinlock.text" => 1, 135 ".spinlock.text" => 1,
136 ".irqentry.text" => 1, 136 ".irqentry.text" => 1,
137 ".kprobes.text" => 1,
137 ".text.unlikely" => 1, 138 ".text.unlikely" => 1,
138); 139);
139 140
@@ -222,6 +223,7 @@ if ($arch eq "x86_64") {
222 $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\smcount([+-]0x[0-9a-zA-Z]+)?\$"; 223 $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\smcount([+-]0x[0-9a-zA-Z]+)?\$";
223 $type = ".quad"; 224 $type = ".quad";
224 $alignment = 8; 225 $alignment = 8;
226 $mcount_adjust = -1;
225 227
226 # force flags for this arch 228 # force flags for this arch
227 $ld .= " -m elf_x86_64"; 229 $ld .= " -m elf_x86_64";
@@ -231,6 +233,7 @@ if ($arch eq "x86_64") {
231 233
232} elsif ($arch eq "i386") { 234} elsif ($arch eq "i386") {
233 $alignment = 4; 235 $alignment = 4;
236 $mcount_adjust = -1;
234 237
235 # force flags for this arch 238 # force flags for this arch
236 $ld .= " -m elf_i386"; 239 $ld .= " -m elf_i386";
@@ -240,12 +243,14 @@ if ($arch eq "x86_64") {
240 243
241} elsif ($arch eq "s390" && $bits == 32) { 244} elsif ($arch eq "s390" && $bits == 32) {
242 $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_32\\s+_mcount\$"; 245 $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_32\\s+_mcount\$";
246 $mcount_adjust = -4;
243 $alignment = 4; 247 $alignment = 4;
244 $ld .= " -m elf_s390"; 248 $ld .= " -m elf_s390";
245 $cc .= " -m31"; 249 $cc .= " -m31";
246 250
247} elsif ($arch eq "s390" && $bits == 64) { 251} elsif ($arch eq "s390" && $bits == 64) {
248 $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_(PC|PLT)32DBL\\s+_mcount\\+0x2\$"; 252 $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_(PC|PLT)32DBL\\s+_mcount\\+0x2\$";
253 $mcount_adjust = -8;
249 $alignment = 8; 254 $alignment = 8;
250 $type = ".quad"; 255 $type = ".quad";
251 $ld .= " -m elf64_s390"; 256 $ld .= " -m elf64_s390";
diff --git a/tools/perf/Documentation/perf-script.txt b/tools/perf/Documentation/perf-script.txt
index 66f040b30729..86c87e214b11 100644
--- a/tools/perf/Documentation/perf-script.txt
+++ b/tools/perf/Documentation/perf-script.txt
@@ -113,13 +113,61 @@ OPTIONS
113 Do various checks like samples ordering and lost events. 113 Do various checks like samples ordering and lost events.
114 114
115-f:: 115-f::
116--fields 116--fields::
117 Comma separated list of fields to print. Options are: 117 Comma separated list of fields to print. Options are:
118 comm, tid, pid, time, cpu, event, trace, sym. Field 118 comm, tid, pid, time, cpu, event, trace, sym. Field
119 list must be prepended with the type, trace, sw or hw, 119 list can be prepended with the type, trace, sw or hw,
120 to indicate to which event type the field list applies. 120 to indicate to which event type the field list applies.
121 e.g., -f sw:comm,tid,time,sym and -f trace:time,cpu,trace 121 e.g., -f sw:comm,tid,time,sym and -f trace:time,cpu,trace
122 122
123 perf script -f <fields>
124
125 is equivalent to:
126
127 perf script -f trace:<fields> -f sw:<fields> -f hw:<fields>
128
129 i.e., the specified fields apply to all event types if the type string
130 is not given.
131
132 The arguments are processed in the order received. A later usage can
133 reset a prior request. e.g.:
134
135 -f trace: -f comm,tid,time,sym
136
137 The first -f suppresses trace events (field list is ""), but then the
138 second invocation sets the fields to comm,tid,time,sym. In this case a
139 warning is given to the user:
140
141 "Overriding previous field request for all events."
142
143 Alternativey, consider the order:
144
145 -f comm,tid,time,sym -f trace:
146
147 The first -f sets the fields for all events and the second -f
148 suppresses trace events. The user is given a warning message about
149 the override, and the result of the above is that only S/W and H/W
150 events are displayed with the given fields.
151
152 For the 'wildcard' option if a user selected field is invalid for an
153 event type, a message is displayed to the user that the option is
154 ignored for that type. For example:
155
156 $ perf script -f comm,tid,trace
157 'trace' not valid for hardware events. Ignoring.
158 'trace' not valid for software events. Ignoring.
159
160 Alternatively, if the type is given an invalid field is specified it
161 is an error. For example:
162
163 perf script -v -f sw:comm,tid,trace
164 'trace' not valid for software events.
165
166 At this point usage is displayed, and perf-script exits.
167
168 Finally, a user may not set fields to none for all event types.
169 i.e., -f "" is not allowed.
170
123-k:: 171-k::
124--vmlinux=<file>:: 172--vmlinux=<file>::
125 vmlinux pathname 173 vmlinux pathname
diff --git a/tools/perf/Makefile b/tools/perf/Makefile
index 0c542563ea6c..1455413ec7a7 100644
--- a/tools/perf/Makefile
+++ b/tools/perf/Makefile
@@ -5,6 +5,8 @@ endif
5# The default target of this Makefile is... 5# The default target of this Makefile is...
6all: 6all:
7 7
8include config/utilities.mak
9
8ifneq ($(OUTPUT),) 10ifneq ($(OUTPUT),)
9# check that the output directory actually exists 11# check that the output directory actually exists
10OUTDIR := $(shell cd $(OUTPUT) && /bin/pwd) 12OUTDIR := $(shell cd $(OUTPUT) && /bin/pwd)
@@ -13,6 +15,12 @@ endif
13 15
14# Define V to have a more verbose compile. 16# Define V to have a more verbose compile.
15# 17#
18# Define PYTHON to point to the python binary if the default
19# `python' is not correct; for example: PYTHON=python2
20#
21# Define PYTHON_CONFIG to point to the python-config binary if
22# the default `$(PYTHON)-config' is not correct.
23#
16# Define ASCIIDOC8 if you want to format documentation with AsciiDoc 8 24# Define ASCIIDOC8 if you want to format documentation with AsciiDoc 8
17# 25#
18# Define DOCBOOK_XSL_172 if you want to format man pages with DocBook XSL v1.72. 26# Define DOCBOOK_XSL_172 if you want to format man pages with DocBook XSL v1.72.
@@ -134,7 +142,7 @@ INSTALL = install
134# explicitly what architecture to check for. Fix this up for yours.. 142# explicitly what architecture to check for. Fix this up for yours..
135SPARSE_FLAGS = -D__BIG_ENDIAN__ -D__powerpc__ 143SPARSE_FLAGS = -D__BIG_ENDIAN__ -D__powerpc__
136 144
137-include feature-tests.mak 145-include config/feature-tests.mak
138 146
139ifeq ($(call try-cc,$(SOURCE_HELLO),-Werror -fstack-protector-all),y) 147ifeq ($(call try-cc,$(SOURCE_HELLO),-Werror -fstack-protector-all),y)
140 CFLAGS := $(CFLAGS) -fstack-protector-all 148 CFLAGS := $(CFLAGS) -fstack-protector-all
@@ -169,12 +177,10 @@ grep-libs = $(filter -l%,$(1))
169strip-libs = $(filter-out -l%,$(1)) 177strip-libs = $(filter-out -l%,$(1))
170 178
171$(OUTPUT)python/perf.so: $(PYRF_OBJS) 179$(OUTPUT)python/perf.so: $(PYRF_OBJS)
172 $(QUIET_GEN)( \ 180 $(QUIET_GEN)CFLAGS='$(BASIC_CFLAGS)' $(PYTHON_WORD) util/setup.py \
173 export CFLAGS="$(BASIC_CFLAGS)"; \ 181 --quiet build_ext \
174 python util/setup.py --quiet build_ext --build-lib='$(OUTPUT)python' \ 182 --build-lib='$(OUTPUT)python' \
175 --build-temp='$(OUTPUT)python/temp' \ 183 --build-temp='$(OUTPUT)python/temp'
176 )
177
178# 184#
179# No Perl scripts right now: 185# No Perl scripts right now:
180# 186#
@@ -479,24 +485,74 @@ else
479 endif 485 endif
480endif 486endif
481 487
482ifdef NO_LIBPYTHON 488disable-python = $(eval $(disable-python_code))
483 BASIC_CFLAGS += -DNO_LIBPYTHON 489define disable-python_code
490 BASIC_CFLAGS += -DNO_LIBPYTHON
491 $(if $(1),$(warning No $(1) was found))
492 $(warning Python support won't be built)
493endef
494
495override PYTHON := \
496 $(call get-executable-or-default,PYTHON,python)
497
498ifndef PYTHON
499 $(call disable-python,python interpreter)
500 python-clean :=
484else 501else
485 PYTHON_EMBED_LDOPTS = $(shell python-config --ldflags 2>/dev/null) 502
486 PYTHON_EMBED_LDFLAGS = $(call strip-libs,$(PYTHON_EMBED_LDOPTS)) 503 PYTHON_WORD := $(call shell-wordify,$(PYTHON))
487 PYTHON_EMBED_LIBADD = $(call grep-libs,$(PYTHON_EMBED_LDOPTS)) 504
488 PYTHON_EMBED_CCOPTS = `python-config --cflags 2>/dev/null` 505 python-clean := $(PYTHON_WORD) util/setup.py clean \
489 FLAGS_PYTHON_EMBED=$(PYTHON_EMBED_CCOPTS) $(PYTHON_EMBED_LDOPTS) 506 --build-lib='$(OUTPUT)python' \
490 ifneq ($(call try-cc,$(SOURCE_PYTHON_EMBED),$(FLAGS_PYTHON_EMBED)),y) 507 --build-temp='$(OUTPUT)python/temp'
491 msg := $(warning No Python.h found, install python-dev[el] to have python support in 'perf script' and to build the python bindings) 508
492 BASIC_CFLAGS += -DNO_LIBPYTHON 509 ifdef NO_LIBPYTHON
493 else 510 $(call disable-python)
494 ALL_LDFLAGS += $(PYTHON_EMBED_LDFLAGS) 511 else
495 EXTLIBS += $(PYTHON_EMBED_LIBADD) 512
496 LIB_OBJS += $(OUTPUT)util/scripting-engines/trace-event-python.o 513 override PYTHON_CONFIG := \
497 LIB_OBJS += $(OUTPUT)scripts/python/Perf-Trace-Util/Context.o 514 $(call get-executable-or-default,PYTHON_CONFIG,$(PYTHON)-config)
498 LANG_BINDINGS += $(OUTPUT)python/perf.so 515
499 endif 516 ifndef PYTHON_CONFIG
517 $(call disable-python,python-config tool)
518 else
519
520 PYTHON_CONFIG_SQ := $(call shell-sq,$(PYTHON_CONFIG))
521
522 PYTHON_EMBED_LDOPTS := $(shell $(PYTHON_CONFIG_SQ) --ldflags 2>/dev/null)
523 PYTHON_EMBED_LDFLAGS := $(call strip-libs,$(PYTHON_EMBED_LDOPTS))
524 PYTHON_EMBED_LIBADD := $(call grep-libs,$(PYTHON_EMBED_LDOPTS))
525 PYTHON_EMBED_CCOPTS := $(shell $(PYTHON_CONFIG_SQ) --cflags 2>/dev/null)
526 FLAGS_PYTHON_EMBED := $(PYTHON_EMBED_CCOPTS) $(PYTHON_EMBED_LDOPTS)
527
528 ifneq ($(call try-cc,$(SOURCE_PYTHON_EMBED),$(FLAGS_PYTHON_EMBED)),y)
529 $(call disable-python,Python.h (for Python 2.x))
530 else
531
532 ifneq ($(call try-cc,$(SOURCE_PYTHON_VERSION),$(FLAGS_PYTHON_EMBED)),y)
533 $(warning Python 3 is not yet supported; please set)
534 $(warning PYTHON and/or PYTHON_CONFIG appropriately.)
535 $(warning If you also have Python 2 installed, then)
536 $(warning try something like:)
537 $(warning $(and ,))
538 $(warning $(and ,) make PYTHON=python2)
539 $(warning $(and ,))
540 $(warning Otherwise, disable Python support entirely:)
541 $(warning $(and ,))
542 $(warning $(and ,) make NO_LIBPYTHON=1)
543 $(warning $(and ,))
544 $(error $(and ,))
545 else
546 ALL_LDFLAGS += $(PYTHON_EMBED_LDFLAGS)
547 EXTLIBS += $(PYTHON_EMBED_LIBADD)
548 LIB_OBJS += $(OUTPUT)util/scripting-engines/trace-event-python.o
549 LIB_OBJS += $(OUTPUT)scripts/python/Perf-Trace-Util/Context.o
550 LANG_BINDINGS += $(OUTPUT)python/perf.so
551 endif
552
553 endif
554 endif
555 endif
500endif 556endif
501 557
502ifdef NO_DEMANGLE 558ifdef NO_DEMANGLE
@@ -837,8 +893,7 @@ clean:
837 $(RM) *.spec *.pyc *.pyo */*.pyc */*.pyo $(OUTPUT)common-cmds.h TAGS tags cscope* 893 $(RM) *.spec *.pyc *.pyo */*.pyc */*.pyo $(OUTPUT)common-cmds.h TAGS tags cscope*
838 $(MAKE) -C Documentation/ clean 894 $(MAKE) -C Documentation/ clean
839 $(RM) $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)PERF-CFLAGS 895 $(RM) $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)PERF-CFLAGS
840 @python util/setup.py clean --build-lib='$(OUTPUT)python' \ 896 $(python-clean)
841 --build-temp='$(OUTPUT)python/temp'
842 897
843.PHONY: all install clean strip 898.PHONY: all install clean strip
844.PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell 899.PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell
diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c
index ac574ea23917..974f6d3f4e53 100644
--- a/tools/perf/builtin-script.c
+++ b/tools/perf/builtin-script.c
@@ -49,57 +49,169 @@ struct output_option {
49}; 49};
50 50
51/* default set to maintain compatibility with current format */ 51/* default set to maintain compatibility with current format */
52static u64 output_fields[PERF_TYPE_MAX] = { 52static struct {
53 [PERF_TYPE_HARDWARE] = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | \ 53 bool user_set;
54 PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | \ 54 bool wildcard_set;
55 PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM, 55 u64 fields;
56 56 u64 invalid_fields;
57 [PERF_TYPE_SOFTWARE] = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | \ 57} output[PERF_TYPE_MAX] = {
58 PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | \ 58
59 PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM, 59 [PERF_TYPE_HARDWARE] = {
60 60 .user_set = false,
61 [PERF_TYPE_TRACEPOINT] = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | \ 61
62 PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | \ 62 .fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID |
63 PERF_OUTPUT_EVNAME | PERF_OUTPUT_TRACE, 63 PERF_OUTPUT_CPU | PERF_OUTPUT_TIME |
64 PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM,
65
66 .invalid_fields = PERF_OUTPUT_TRACE,
67 },
68
69 [PERF_TYPE_SOFTWARE] = {
70 .user_set = false,
71
72 .fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID |
73 PERF_OUTPUT_CPU | PERF_OUTPUT_TIME |
74 PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM,
75
76 .invalid_fields = PERF_OUTPUT_TRACE,
77 },
78
79 [PERF_TYPE_TRACEPOINT] = {
80 .user_set = false,
81
82 .fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID |
83 PERF_OUTPUT_CPU | PERF_OUTPUT_TIME |
84 PERF_OUTPUT_EVNAME | PERF_OUTPUT_TRACE,
85 },
86
87 [PERF_TYPE_RAW] = {
88 .user_set = false,
89
90 .fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID |
91 PERF_OUTPUT_CPU | PERF_OUTPUT_TIME |
92 PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM,
93
94 .invalid_fields = PERF_OUTPUT_TRACE,
95 },
64}; 96};
65 97
66static bool output_set_by_user; 98static bool output_set_by_user(void)
99{
100 int j;
101 for (j = 0; j < PERF_TYPE_MAX; ++j) {
102 if (output[j].user_set)
103 return true;
104 }
105 return false;
106}
107
108static const char *output_field2str(enum perf_output_field field)
109{
110 int i, imax = ARRAY_SIZE(all_output_options);
111 const char *str = "";
112
113 for (i = 0; i < imax; ++i) {
114 if (all_output_options[i].field == field) {
115 str = all_output_options[i].str;
116 break;
117 }
118 }
119 return str;
120}
67 121
68#define PRINT_FIELD(x) (output_fields[attr->type] & PERF_OUTPUT_##x) 122#define PRINT_FIELD(x) (output[attr->type].fields & PERF_OUTPUT_##x)
69 123
70static int perf_session__check_attr(struct perf_session *session, 124static int perf_event_attr__check_stype(struct perf_event_attr *attr,
71 struct perf_event_attr *attr) 125 u64 sample_type, const char *sample_msg,
126 enum perf_output_field field)
72{ 127{
128 int type = attr->type;
129 const char *evname;
130
131 if (attr->sample_type & sample_type)
132 return 0;
133
134 if (output[type].user_set) {
135 evname = __event_name(attr->type, attr->config);
136 pr_err("Samples for '%s' event do not have %s attribute set. "
137 "Cannot print '%s' field.\n",
138 evname, sample_msg, output_field2str(field));
139 return -1;
140 }
141
142 /* user did not ask for it explicitly so remove from the default list */
143 output[type].fields &= ~field;
144 evname = __event_name(attr->type, attr->config);
145 pr_debug("Samples for '%s' event do not have %s attribute set. "
146 "Skipping '%s' field.\n",
147 evname, sample_msg, output_field2str(field));
148
149 return 0;
150}
151
152static int perf_evsel__check_attr(struct perf_evsel *evsel,
153 struct perf_session *session)
154{
155 struct perf_event_attr *attr = &evsel->attr;
156
73 if (PRINT_FIELD(TRACE) && 157 if (PRINT_FIELD(TRACE) &&
74 !perf_session__has_traces(session, "record -R")) 158 !perf_session__has_traces(session, "record -R"))
75 return -EINVAL; 159 return -EINVAL;
76 160
77 if (PRINT_FIELD(SYM)) { 161 if (PRINT_FIELD(SYM)) {
78 if (!(session->sample_type & PERF_SAMPLE_IP)) { 162 if (perf_event_attr__check_stype(attr, PERF_SAMPLE_IP, "IP",
79 pr_err("Samples do not contain IP data.\n"); 163 PERF_OUTPUT_SYM))
80 return -EINVAL; 164 return -EINVAL;
81 } 165
82 if (!no_callchain && 166 if (!no_callchain &&
83 !(session->sample_type & PERF_SAMPLE_CALLCHAIN)) 167 !(attr->sample_type & PERF_SAMPLE_CALLCHAIN))
84 symbol_conf.use_callchain = false; 168 symbol_conf.use_callchain = false;
85 } 169 }
86 170
87 if ((PRINT_FIELD(PID) || PRINT_FIELD(TID)) && 171 if ((PRINT_FIELD(PID) || PRINT_FIELD(TID)) &&
88 !(session->sample_type & PERF_SAMPLE_TID)) { 172 perf_event_attr__check_stype(attr, PERF_SAMPLE_TID, "TID",
89 pr_err("Samples do not contain TID/PID data.\n"); 173 PERF_OUTPUT_TID|PERF_OUTPUT_PID))
90 return -EINVAL; 174 return -EINVAL;
91 }
92 175
93 if (PRINT_FIELD(TIME) && 176 if (PRINT_FIELD(TIME) &&
94 !(session->sample_type & PERF_SAMPLE_TIME)) { 177 perf_event_attr__check_stype(attr, PERF_SAMPLE_TIME, "TIME",
95 pr_err("Samples do not contain timestamps.\n"); 178 PERF_OUTPUT_TIME))
96 return -EINVAL; 179 return -EINVAL;
97 }
98 180
99 if (PRINT_FIELD(CPU) && 181 if (PRINT_FIELD(CPU) &&
100 !(session->sample_type & PERF_SAMPLE_CPU)) { 182 perf_event_attr__check_stype(attr, PERF_SAMPLE_CPU, "CPU",
101 pr_err("Samples do not contain cpu.\n"); 183 PERF_OUTPUT_CPU))
102 return -EINVAL; 184 return -EINVAL;
185
186 return 0;
187}
188
189/*
190 * verify all user requested events exist and the samples
191 * have the expected data
192 */
193static int perf_session__check_output_opt(struct perf_session *session)
194{
195 int j;
196 struct perf_evsel *evsel;
197
198 for (j = 0; j < PERF_TYPE_MAX; ++j) {
199 evsel = perf_session__find_first_evtype(session, j);
200
201 /*
202 * even if fields is set to 0 (ie., show nothing) event must
203 * exist if user explicitly includes it on the command line
204 */
205 if (!evsel && output[j].user_set && !output[j].wildcard_set) {
206 pr_err("%s events do not exist. "
207 "Remove corresponding -f option to proceed.\n",
208 event_type(j));
209 return -1;
210 }
211
212 if (evsel && output[j].fields &&
213 perf_evsel__check_attr(evsel, session))
214 return -1;
103 } 215 }
104 216
105 return 0; 217 return 0;
@@ -168,10 +280,7 @@ static void process_event(union perf_event *event __unused,
168{ 280{
169 struct perf_event_attr *attr = &evsel->attr; 281 struct perf_event_attr *attr = &evsel->attr;
170 282
171 if (output_fields[attr->type] == 0) 283 if (output[attr->type].fields == 0)
172 return;
173
174 if (perf_session__check_attr(session, attr) < 0)
175 return; 284 return;
176 285
177 print_sample_start(sample, thread, attr); 286 print_sample_start(sample, thread, attr);
@@ -451,6 +560,7 @@ static int parse_output_fields(const struct option *opt __used,
451{ 560{
452 char *tok; 561 char *tok;
453 int i, imax = sizeof(all_output_options) / sizeof(struct output_option); 562 int i, imax = sizeof(all_output_options) / sizeof(struct output_option);
563 int j;
454 int rc = 0; 564 int rc = 0;
455 char *str = strdup(arg); 565 char *str = strdup(arg);
456 int type = -1; 566 int type = -1;
@@ -458,52 +568,99 @@ static int parse_output_fields(const struct option *opt __used,
458 if (!str) 568 if (!str)
459 return -ENOMEM; 569 return -ENOMEM;
460 570
461 tok = strtok(str, ":"); 571 /* first word can state for which event type the user is specifying
462 if (!tok) { 572 * the fields. If no type exists, the specified fields apply to all
463 fprintf(stderr, 573 * event types found in the file minus the invalid fields for a type.
464 "Invalid field string - not prepended with type.");
465 return -EINVAL;
466 }
467
468 /* first word should state which event type user
469 * is specifying the fields
470 */ 574 */
471 if (!strcmp(tok, "hw")) 575 tok = strchr(str, ':');
472 type = PERF_TYPE_HARDWARE; 576 if (tok) {
473 else if (!strcmp(tok, "sw")) 577 *tok = '\0';
474 type = PERF_TYPE_SOFTWARE; 578 tok++;
475 else if (!strcmp(tok, "trace")) 579 if (!strcmp(str, "hw"))
476 type = PERF_TYPE_TRACEPOINT; 580 type = PERF_TYPE_HARDWARE;
477 else { 581 else if (!strcmp(str, "sw"))
478 fprintf(stderr, "Invalid event type in field string."); 582 type = PERF_TYPE_SOFTWARE;
479 return -EINVAL; 583 else if (!strcmp(str, "trace"))
584 type = PERF_TYPE_TRACEPOINT;
585 else if (!strcmp(str, "raw"))
586 type = PERF_TYPE_RAW;
587 else {
588 fprintf(stderr, "Invalid event type in field string.\n");
589 return -EINVAL;
590 }
591
592 if (output[type].user_set)
593 pr_warning("Overriding previous field request for %s events.\n",
594 event_type(type));
595
596 output[type].fields = 0;
597 output[type].user_set = true;
598 output[type].wildcard_set = false;
599
600 } else {
601 tok = str;
602 if (strlen(str) == 0) {
603 fprintf(stderr,
604 "Cannot set fields to 'none' for all event types.\n");
605 rc = -EINVAL;
606 goto out;
607 }
608
609 if (output_set_by_user())
610 pr_warning("Overriding previous field request for all events.\n");
611
612 for (j = 0; j < PERF_TYPE_MAX; ++j) {
613 output[j].fields = 0;
614 output[j].user_set = true;
615 output[j].wildcard_set = true;
616 }
480 } 617 }
481 618
482 output_fields[type] = 0; 619 tok = strtok(tok, ",");
483 while (1) { 620 while (tok) {
484 tok = strtok(NULL, ",");
485 if (!tok)
486 break;
487 for (i = 0; i < imax; ++i) { 621 for (i = 0; i < imax; ++i) {
488 if (strcmp(tok, all_output_options[i].str) == 0) { 622 if (strcmp(tok, all_output_options[i].str) == 0)
489 output_fields[type] |= all_output_options[i].field;
490 break; 623 break;
491 }
492 } 624 }
493 if (i == imax) { 625 if (i == imax) {
494 fprintf(stderr, "Invalid field requested."); 626 fprintf(stderr, "Invalid field requested.\n");
495 rc = -EINVAL; 627 rc = -EINVAL;
496 break; 628 goto out;
497 } 629 }
498 }
499 630
500 if (output_fields[type] == 0) { 631 if (type == -1) {
501 pr_debug("No fields requested for %s type. " 632 /* add user option to all events types for
502 "Events will not be displayed\n", event_type(type)); 633 * which it is valid
634 */
635 for (j = 0; j < PERF_TYPE_MAX; ++j) {
636 if (output[j].invalid_fields & all_output_options[i].field) {
637 pr_warning("\'%s\' not valid for %s events. Ignoring.\n",
638 all_output_options[i].str, event_type(j));
639 } else
640 output[j].fields |= all_output_options[i].field;
641 }
642 } else {
643 if (output[type].invalid_fields & all_output_options[i].field) {
644 fprintf(stderr, "\'%s\' not valid for %s events.\n",
645 all_output_options[i].str, event_type(type));
646
647 rc = -EINVAL;
648 goto out;
649 }
650 output[type].fields |= all_output_options[i].field;
651 }
652
653 tok = strtok(NULL, ",");
503 } 654 }
504 655
505 output_set_by_user = true; 656 if (type >= 0) {
657 if (output[type].fields == 0) {
658 pr_debug("No fields requested for %s type. "
659 "Events will not be displayed.\n", event_type(type));
660 }
661 }
506 662
663out:
507 free(str); 664 free(str);
508 return rc; 665 return rc;
509} 666}
@@ -829,7 +986,7 @@ static const struct option options[] = {
829 OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory", 986 OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
830 "Look for files with symbols relative to this directory"), 987 "Look for files with symbols relative to this directory"),
831 OPT_CALLBACK('f', "fields", NULL, "str", 988 OPT_CALLBACK('f', "fields", NULL, "str",
832 "comma separated output fields prepend with 'type:'. Valid types: hw,sw,trace. Fields: comm,tid,pid,time,cpu,event,trace,sym", 989 "comma separated output fields prepend with 'type:'. Valid types: hw,sw,trace,raw. Fields: comm,tid,pid,time,cpu,event,trace,sym",
833 parse_output_fields), 990 parse_output_fields),
834 991
835 OPT_END() 992 OPT_END()
@@ -1020,7 +1177,7 @@ int cmd_script(int argc, const char **argv, const char *prefix __used)
1020 struct stat perf_stat; 1177 struct stat perf_stat;
1021 int input; 1178 int input;
1022 1179
1023 if (output_set_by_user) { 1180 if (output_set_by_user()) {
1024 fprintf(stderr, 1181 fprintf(stderr,
1025 "custom fields not supported for generated scripts"); 1182 "custom fields not supported for generated scripts");
1026 return -1; 1183 return -1;
@@ -1060,6 +1217,11 @@ int cmd_script(int argc, const char **argv, const char *prefix __used)
1060 pr_debug("perf script started with script %s\n\n", script_name); 1217 pr_debug("perf script started with script %s\n\n", script_name);
1061 } 1218 }
1062 1219
1220
1221 err = perf_session__check_output_opt(session);
1222 if (err < 0)
1223 goto out;
1224
1063 err = __cmd_script(session); 1225 err = __cmd_script(session);
1064 1226
1065 perf_session__delete(session); 1227 perf_session__delete(session);
diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
index 03f0e45f1479..a9f06715e44d 100644
--- a/tools/perf/builtin-stat.c
+++ b/tools/perf/builtin-stat.c
@@ -6,24 +6,28 @@
6 * 6 *
7 * Sample output: 7 * Sample output:
8 8
9 $ perf stat ~/hackbench 10 9 $ perf stat ./hackbench 10
10 Time: 0.104
11 10
12 Performance counter stats for '/home/mingo/hackbench': 11 Time: 0.118
13 12
14 1255.538611 task clock ticks # 10.143 CPU utilization factor 13 Performance counter stats for './hackbench 10':
15 54011 context switches # 0.043 M/sec
16 385 CPU migrations # 0.000 M/sec
17 17755 pagefaults # 0.014 M/sec
18 3808323185 CPU cycles # 3033.219 M/sec
19 1575111190 instructions # 1254.530 M/sec
20 17367895 cache references # 13.833 M/sec
21 7674421 cache misses # 6.112 M/sec
22 14
23 Wall-clock time elapsed: 123.786620 msecs 15 1708.761321 task-clock # 11.037 CPUs utilized
16 41,190 context-switches # 0.024 M/sec
17 6,735 CPU-migrations # 0.004 M/sec
18 17,318 page-faults # 0.010 M/sec
19 5,205,202,243 cycles # 3.046 GHz
20 3,856,436,920 stalled-cycles-frontend # 74.09% frontend cycles idle
21 1,600,790,871 stalled-cycles-backend # 30.75% backend cycles idle
22 2,603,501,247 instructions # 0.50 insns per cycle
23 # 1.48 stalled cycles per insn
24 484,357,498 branches # 283.455 M/sec
25 6,388,934 branch-misses # 1.32% of all branches
26
27 0.154822978 seconds time elapsed
24 28
25 * 29 *
26 * Copyright (C) 2008, Red Hat Inc, Ingo Molnar <mingo@redhat.com> 30 * Copyright (C) 2008-2011, Red Hat Inc, Ingo Molnar <mingo@redhat.com>
27 * 31 *
28 * Improvements and fixes by: 32 * Improvements and fixes by:
29 * 33 *
@@ -46,6 +50,7 @@
46#include "util/evlist.h" 50#include "util/evlist.h"
47#include "util/evsel.h" 51#include "util/evsel.h"
48#include "util/debug.h" 52#include "util/debug.h"
53#include "util/color.h"
49#include "util/header.h" 54#include "util/header.h"
50#include "util/cpumap.h" 55#include "util/cpumap.h"
51#include "util/thread.h" 56#include "util/thread.h"
@@ -65,14 +70,107 @@ static struct perf_event_attr default_attrs[] = {
65 { .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_PAGE_FAULTS }, 70 { .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_PAGE_FAULTS },
66 71
67 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES }, 72 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES },
73 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_STALLED_CYCLES_FRONTEND },
74 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_STALLED_CYCLES_BACKEND },
68 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS }, 75 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS },
69 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, 76 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS },
70 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_MISSES }, 77 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_MISSES },
71 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CACHE_REFERENCES },
72 { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CACHE_MISSES },
73 78
74}; 79};
75 80
81/*
82 * Detailed stats (-d), covering the L1 and last level data caches:
83 */
84static struct perf_event_attr detailed_attrs[] = {
85
86 { .type = PERF_TYPE_HW_CACHE,
87 .config =
88 PERF_COUNT_HW_CACHE_L1D << 0 |
89 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
90 (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
91
92 { .type = PERF_TYPE_HW_CACHE,
93 .config =
94 PERF_COUNT_HW_CACHE_L1D << 0 |
95 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
96 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
97
98 { .type = PERF_TYPE_HW_CACHE,
99 .config =
100 PERF_COUNT_HW_CACHE_LL << 0 |
101 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
102 (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
103
104 { .type = PERF_TYPE_HW_CACHE,
105 .config =
106 PERF_COUNT_HW_CACHE_LL << 0 |
107 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
108 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
109};
110
111/*
112 * Very detailed stats (-d -d), covering the instruction cache and the TLB caches:
113 */
114static struct perf_event_attr very_detailed_attrs[] = {
115
116 { .type = PERF_TYPE_HW_CACHE,
117 .config =
118 PERF_COUNT_HW_CACHE_L1I << 0 |
119 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
120 (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
121
122 { .type = PERF_TYPE_HW_CACHE,
123 .config =
124 PERF_COUNT_HW_CACHE_L1I << 0 |
125 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
126 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
127
128 { .type = PERF_TYPE_HW_CACHE,
129 .config =
130 PERF_COUNT_HW_CACHE_DTLB << 0 |
131 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
132 (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
133
134 { .type = PERF_TYPE_HW_CACHE,
135 .config =
136 PERF_COUNT_HW_CACHE_DTLB << 0 |
137 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
138 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
139
140 { .type = PERF_TYPE_HW_CACHE,
141 .config =
142 PERF_COUNT_HW_CACHE_ITLB << 0 |
143 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
144 (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
145
146 { .type = PERF_TYPE_HW_CACHE,
147 .config =
148 PERF_COUNT_HW_CACHE_ITLB << 0 |
149 (PERF_COUNT_HW_CACHE_OP_READ << 8) |
150 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
151
152};
153
154/*
155 * Very, very detailed stats (-d -d -d), adding prefetch events:
156 */
157static struct perf_event_attr very_very_detailed_attrs[] = {
158
159 { .type = PERF_TYPE_HW_CACHE,
160 .config =
161 PERF_COUNT_HW_CACHE_L1D << 0 |
162 (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) |
163 (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
164
165 { .type = PERF_TYPE_HW_CACHE,
166 .config =
167 PERF_COUNT_HW_CACHE_L1D << 0 |
168 (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) |
169 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
170};
171
172
173
76struct perf_evlist *evsel_list; 174struct perf_evlist *evsel_list;
77 175
78static bool system_wide = false; 176static bool system_wide = false;
@@ -86,6 +184,8 @@ static pid_t target_pid = -1;
86static pid_t target_tid = -1; 184static pid_t target_tid = -1;
87static pid_t child_pid = -1; 185static pid_t child_pid = -1;
88static bool null_run = false; 186static bool null_run = false;
187static int detailed_run = 0;
188static bool sync_run = false;
89static bool big_num = true; 189static bool big_num = true;
90static int big_num_opt = -1; 190static int big_num_opt = -1;
91static const char *cpu_list; 191static const char *cpu_list;
@@ -156,7 +256,15 @@ static double stddev_stats(struct stats *stats)
156 256
157struct stats runtime_nsecs_stats[MAX_NR_CPUS]; 257struct stats runtime_nsecs_stats[MAX_NR_CPUS];
158struct stats runtime_cycles_stats[MAX_NR_CPUS]; 258struct stats runtime_cycles_stats[MAX_NR_CPUS];
259struct stats runtime_stalled_cycles_front_stats[MAX_NR_CPUS];
260struct stats runtime_stalled_cycles_back_stats[MAX_NR_CPUS];
159struct stats runtime_branches_stats[MAX_NR_CPUS]; 261struct stats runtime_branches_stats[MAX_NR_CPUS];
262struct stats runtime_cacherefs_stats[MAX_NR_CPUS];
263struct stats runtime_l1_dcache_stats[MAX_NR_CPUS];
264struct stats runtime_l1_icache_stats[MAX_NR_CPUS];
265struct stats runtime_ll_cache_stats[MAX_NR_CPUS];
266struct stats runtime_itlb_cache_stats[MAX_NR_CPUS];
267struct stats runtime_dtlb_cache_stats[MAX_NR_CPUS];
160struct stats walltime_nsecs_stats; 268struct stats walltime_nsecs_stats;
161 269
162static int create_perf_stat_counter(struct perf_evsel *evsel) 270static int create_perf_stat_counter(struct perf_evsel *evsel)
@@ -193,6 +301,37 @@ static inline int nsec_counter(struct perf_evsel *evsel)
193} 301}
194 302
195/* 303/*
304 * Update various tracking values we maintain to print
305 * more semantic information such as miss/hit ratios,
306 * instruction rates, etc:
307 */
308static void update_shadow_stats(struct perf_evsel *counter, u64 *count)
309{
310 if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK))
311 update_stats(&runtime_nsecs_stats[0], count[0]);
312 else if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES))
313 update_stats(&runtime_cycles_stats[0], count[0]);
314 else if (perf_evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_FRONTEND))
315 update_stats(&runtime_stalled_cycles_front_stats[0], count[0]);
316 else if (perf_evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_BACKEND))
317 update_stats(&runtime_stalled_cycles_back_stats[0], count[0]);
318 else if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS))
319 update_stats(&runtime_branches_stats[0], count[0]);
320 else if (perf_evsel__match(counter, HARDWARE, HW_CACHE_REFERENCES))
321 update_stats(&runtime_cacherefs_stats[0], count[0]);
322 else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_L1D))
323 update_stats(&runtime_l1_dcache_stats[0], count[0]);
324 else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_L1I))
325 update_stats(&runtime_l1_icache_stats[0], count[0]);
326 else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_LL))
327 update_stats(&runtime_ll_cache_stats[0], count[0]);
328 else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_DTLB))
329 update_stats(&runtime_dtlb_cache_stats[0], count[0]);
330 else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_ITLB))
331 update_stats(&runtime_itlb_cache_stats[0], count[0]);
332}
333
334/*
196 * Read out the results of a single counter: 335 * Read out the results of a single counter:
197 * aggregate counts across CPUs in system-wide mode 336 * aggregate counts across CPUs in system-wide mode
198 */ 337 */
@@ -217,12 +356,7 @@ static int read_counter_aggr(struct perf_evsel *counter)
217 /* 356 /*
218 * Save the full runtime - to allow normalization during printout: 357 * Save the full runtime - to allow normalization during printout:
219 */ 358 */
220 if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK)) 359 update_shadow_stats(counter, count);
221 update_stats(&runtime_nsecs_stats[0], count[0]);
222 if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES))
223 update_stats(&runtime_cycles_stats[0], count[0]);
224 if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS))
225 update_stats(&runtime_branches_stats[0], count[0]);
226 360
227 return 0; 361 return 0;
228} 362}
@@ -242,12 +376,7 @@ static int read_counter(struct perf_evsel *counter)
242 376
243 count = counter->counts->cpu[cpu].values; 377 count = counter->counts->cpu[cpu].values;
244 378
245 if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK)) 379 update_shadow_stats(counter, count);
246 update_stats(&runtime_nsecs_stats[cpu], count[0]);
247 if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES))
248 update_stats(&runtime_cycles_stats[cpu], count[0]);
249 if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS))
250 update_stats(&runtime_branches_stats[cpu], count[0]);
251 } 380 }
252 381
253 return 0; 382 return 0;
@@ -315,13 +444,18 @@ static int run_perf_stat(int argc __used, const char **argv)
315 444
316 list_for_each_entry(counter, &evsel_list->entries, node) { 445 list_for_each_entry(counter, &evsel_list->entries, node) {
317 if (create_perf_stat_counter(counter) < 0) { 446 if (create_perf_stat_counter(counter) < 0) {
318 if (errno == -EPERM || errno == -EACCES) { 447 if (errno == EINVAL || errno == ENOSYS || errno == ENOENT) {
448 if (verbose)
449 ui__warning("%s event is not supported by the kernel.\n",
450 event_name(counter));
451 continue;
452 }
453
454 if (errno == EPERM || errno == EACCES) {
319 error("You may not have permission to collect %sstats.\n" 455 error("You may not have permission to collect %sstats.\n"
320 "\t Consider tweaking" 456 "\t Consider tweaking"
321 " /proc/sys/kernel/perf_event_paranoid or running as root.", 457 " /proc/sys/kernel/perf_event_paranoid or running as root.",
322 system_wide ? "system-wide " : ""); 458 system_wide ? "system-wide " : "");
323 } else if (errno == ENOENT) {
324 error("%s event is not supported. ", event_name(counter));
325 } else { 459 } else {
326 error("open_counter returned with %d (%s). " 460 error("open_counter returned with %d (%s). "
327 "/bin/dmesg may provide additional information.\n", 461 "/bin/dmesg may provide additional information.\n",
@@ -372,6 +506,16 @@ static int run_perf_stat(int argc __used, const char **argv)
372 return WEXITSTATUS(status); 506 return WEXITSTATUS(status);
373} 507}
374 508
509static void print_noise_pct(double total, double avg)
510{
511 double pct = 0.0;
512
513 if (avg)
514 pct = 100.0*total/avg;
515
516 fprintf(stderr, " ( +-%6.2f%% )", pct);
517}
518
375static void print_noise(struct perf_evsel *evsel, double avg) 519static void print_noise(struct perf_evsel *evsel, double avg)
376{ 520{
377 struct perf_stat *ps; 521 struct perf_stat *ps;
@@ -380,15 +524,14 @@ static void print_noise(struct perf_evsel *evsel, double avg)
380 return; 524 return;
381 525
382 ps = evsel->priv; 526 ps = evsel->priv;
383 fprintf(stderr, " ( +- %7.3f%% )", 527 print_noise_pct(stddev_stats(&ps->res_stats[0]), avg);
384 100 * stddev_stats(&ps->res_stats[0]) / avg);
385} 528}
386 529
387static void nsec_printout(int cpu, struct perf_evsel *evsel, double avg) 530static void nsec_printout(int cpu, struct perf_evsel *evsel, double avg)
388{ 531{
389 double msecs = avg / 1e6; 532 double msecs = avg / 1e6;
390 char cpustr[16] = { '\0', }; 533 char cpustr[16] = { '\0', };
391 const char *fmt = csv_output ? "%s%.6f%s%s" : "%s%18.6f%s%-24s"; 534 const char *fmt = csv_output ? "%s%.6f%s%s" : "%s%18.6f%s%-25s";
392 535
393 if (no_aggr) 536 if (no_aggr)
394 sprintf(cpustr, "CPU%*d%s", 537 sprintf(cpustr, "CPU%*d%s",
@@ -404,8 +547,191 @@ static void nsec_printout(int cpu, struct perf_evsel *evsel, double avg)
404 return; 547 return;
405 548
406 if (perf_evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK)) 549 if (perf_evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK))
407 fprintf(stderr, " # %10.3f CPUs ", 550 fprintf(stderr, " # %8.3f CPUs utilized ", avg / avg_stats(&walltime_nsecs_stats));
408 avg / avg_stats(&walltime_nsecs_stats)); 551}
552
553static void print_stalled_cycles_frontend(int cpu, struct perf_evsel *evsel __used, double avg)
554{
555 double total, ratio = 0.0;
556 const char *color;
557
558 total = avg_stats(&runtime_cycles_stats[cpu]);
559
560 if (total)
561 ratio = avg / total * 100.0;
562
563 color = PERF_COLOR_NORMAL;
564 if (ratio > 50.0)
565 color = PERF_COLOR_RED;
566 else if (ratio > 30.0)
567 color = PERF_COLOR_MAGENTA;
568 else if (ratio > 10.0)
569 color = PERF_COLOR_YELLOW;
570
571 fprintf(stderr, " # ");
572 color_fprintf(stderr, color, "%6.2f%%", ratio);
573 fprintf(stderr, " frontend cycles idle ");
574}
575
576static void print_stalled_cycles_backend(int cpu, struct perf_evsel *evsel __used, double avg)
577{
578 double total, ratio = 0.0;
579 const char *color;
580
581 total = avg_stats(&runtime_cycles_stats[cpu]);
582
583 if (total)
584 ratio = avg / total * 100.0;
585
586 color = PERF_COLOR_NORMAL;
587 if (ratio > 75.0)
588 color = PERF_COLOR_RED;
589 else if (ratio > 50.0)
590 color = PERF_COLOR_MAGENTA;
591 else if (ratio > 20.0)
592 color = PERF_COLOR_YELLOW;
593
594 fprintf(stderr, " # ");
595 color_fprintf(stderr, color, "%6.2f%%", ratio);
596 fprintf(stderr, " backend cycles idle ");
597}
598
599static void print_branch_misses(int cpu, struct perf_evsel *evsel __used, double avg)
600{
601 double total, ratio = 0.0;
602 const char *color;
603
604 total = avg_stats(&runtime_branches_stats[cpu]);
605
606 if (total)
607 ratio = avg / total * 100.0;
608
609 color = PERF_COLOR_NORMAL;
610 if (ratio > 20.0)
611 color = PERF_COLOR_RED;
612 else if (ratio > 10.0)
613 color = PERF_COLOR_MAGENTA;
614 else if (ratio > 5.0)
615 color = PERF_COLOR_YELLOW;
616
617 fprintf(stderr, " # ");
618 color_fprintf(stderr, color, "%6.2f%%", ratio);
619 fprintf(stderr, " of all branches ");
620}
621
622static void print_l1_dcache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
623{
624 double total, ratio = 0.0;
625 const char *color;
626
627 total = avg_stats(&runtime_l1_dcache_stats[cpu]);
628
629 if (total)
630 ratio = avg / total * 100.0;
631
632 color = PERF_COLOR_NORMAL;
633 if (ratio > 20.0)
634 color = PERF_COLOR_RED;
635 else if (ratio > 10.0)
636 color = PERF_COLOR_MAGENTA;
637 else if (ratio > 5.0)
638 color = PERF_COLOR_YELLOW;
639
640 fprintf(stderr, " # ");
641 color_fprintf(stderr, color, "%6.2f%%", ratio);
642 fprintf(stderr, " of all L1-dcache hits ");
643}
644
645static void print_l1_icache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
646{
647 double total, ratio = 0.0;
648 const char *color;
649
650 total = avg_stats(&runtime_l1_icache_stats[cpu]);
651
652 if (total)
653 ratio = avg / total * 100.0;
654
655 color = PERF_COLOR_NORMAL;
656 if (ratio > 20.0)
657 color = PERF_COLOR_RED;
658 else if (ratio > 10.0)
659 color = PERF_COLOR_MAGENTA;
660 else if (ratio > 5.0)
661 color = PERF_COLOR_YELLOW;
662
663 fprintf(stderr, " # ");
664 color_fprintf(stderr, color, "%6.2f%%", ratio);
665 fprintf(stderr, " of all L1-icache hits ");
666}
667
668static void print_dtlb_cache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
669{
670 double total, ratio = 0.0;
671 const char *color;
672
673 total = avg_stats(&runtime_dtlb_cache_stats[cpu]);
674
675 if (total)
676 ratio = avg / total * 100.0;
677
678 color = PERF_COLOR_NORMAL;
679 if (ratio > 20.0)
680 color = PERF_COLOR_RED;
681 else if (ratio > 10.0)
682 color = PERF_COLOR_MAGENTA;
683 else if (ratio > 5.0)
684 color = PERF_COLOR_YELLOW;
685
686 fprintf(stderr, " # ");
687 color_fprintf(stderr, color, "%6.2f%%", ratio);
688 fprintf(stderr, " of all dTLB cache hits ");
689}
690
691static void print_itlb_cache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
692{
693 double total, ratio = 0.0;
694 const char *color;
695
696 total = avg_stats(&runtime_itlb_cache_stats[cpu]);
697
698 if (total)
699 ratio = avg / total * 100.0;
700
701 color = PERF_COLOR_NORMAL;
702 if (ratio > 20.0)
703 color = PERF_COLOR_RED;
704 else if (ratio > 10.0)
705 color = PERF_COLOR_MAGENTA;
706 else if (ratio > 5.0)
707 color = PERF_COLOR_YELLOW;
708
709 fprintf(stderr, " # ");
710 color_fprintf(stderr, color, "%6.2f%%", ratio);
711 fprintf(stderr, " of all iTLB cache hits ");
712}
713
714static void print_ll_cache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
715{
716 double total, ratio = 0.0;
717 const char *color;
718
719 total = avg_stats(&runtime_ll_cache_stats[cpu]);
720
721 if (total)
722 ratio = avg / total * 100.0;
723
724 color = PERF_COLOR_NORMAL;
725 if (ratio > 20.0)
726 color = PERF_COLOR_RED;
727 else if (ratio > 10.0)
728 color = PERF_COLOR_MAGENTA;
729 else if (ratio > 5.0)
730 color = PERF_COLOR_YELLOW;
731
732 fprintf(stderr, " # ");
733 color_fprintf(stderr, color, "%6.2f%%", ratio);
734 fprintf(stderr, " of all LL-cache hits ");
409} 735}
410 736
411static void abs_printout(int cpu, struct perf_evsel *evsel, double avg) 737static void abs_printout(int cpu, struct perf_evsel *evsel, double avg)
@@ -417,9 +743,9 @@ static void abs_printout(int cpu, struct perf_evsel *evsel, double avg)
417 if (csv_output) 743 if (csv_output)
418 fmt = "%s%.0f%s%s"; 744 fmt = "%s%.0f%s%s";
419 else if (big_num) 745 else if (big_num)
420 fmt = "%s%'18.0f%s%-24s"; 746 fmt = "%s%'18.0f%s%-25s";
421 else 747 else
422 fmt = "%s%18.0f%s%-24s"; 748 fmt = "%s%18.0f%s%-25s";
423 749
424 if (no_aggr) 750 if (no_aggr)
425 sprintf(cpustr, "CPU%*d%s", 751 sprintf(cpustr, "CPU%*d%s",
@@ -442,23 +768,83 @@ static void abs_printout(int cpu, struct perf_evsel *evsel, double avg)
442 if (total) 768 if (total)
443 ratio = avg / total; 769 ratio = avg / total;
444 770
445 fprintf(stderr, " # %10.3f IPC ", ratio); 771 fprintf(stderr, " # %5.2f insns per cycle ", ratio);
772
773 total = avg_stats(&runtime_stalled_cycles_front_stats[cpu]);
774 total = max(total, avg_stats(&runtime_stalled_cycles_back_stats[cpu]));
775
776 if (total && avg) {
777 ratio = total / avg;
778 fprintf(stderr, "\n # %5.2f stalled cycles per insn", ratio);
779 }
780
446 } else if (perf_evsel__match(evsel, HARDWARE, HW_BRANCH_MISSES) && 781 } else if (perf_evsel__match(evsel, HARDWARE, HW_BRANCH_MISSES) &&
447 runtime_branches_stats[cpu].n != 0) { 782 runtime_branches_stats[cpu].n != 0) {
448 total = avg_stats(&runtime_branches_stats[cpu]); 783 print_branch_misses(cpu, evsel, avg);
784 } else if (
785 evsel->attr.type == PERF_TYPE_HW_CACHE &&
786 evsel->attr.config == ( PERF_COUNT_HW_CACHE_L1D |
787 ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
788 ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
789 runtime_l1_dcache_stats[cpu].n != 0) {
790 print_l1_dcache_misses(cpu, evsel, avg);
791 } else if (
792 evsel->attr.type == PERF_TYPE_HW_CACHE &&
793 evsel->attr.config == ( PERF_COUNT_HW_CACHE_L1I |
794 ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
795 ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
796 runtime_l1_icache_stats[cpu].n != 0) {
797 print_l1_icache_misses(cpu, evsel, avg);
798 } else if (
799 evsel->attr.type == PERF_TYPE_HW_CACHE &&
800 evsel->attr.config == ( PERF_COUNT_HW_CACHE_DTLB |
801 ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
802 ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
803 runtime_dtlb_cache_stats[cpu].n != 0) {
804 print_dtlb_cache_misses(cpu, evsel, avg);
805 } else if (
806 evsel->attr.type == PERF_TYPE_HW_CACHE &&
807 evsel->attr.config == ( PERF_COUNT_HW_CACHE_ITLB |
808 ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
809 ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
810 runtime_itlb_cache_stats[cpu].n != 0) {
811 print_itlb_cache_misses(cpu, evsel, avg);
812 } else if (
813 evsel->attr.type == PERF_TYPE_HW_CACHE &&
814 evsel->attr.config == ( PERF_COUNT_HW_CACHE_LL |
815 ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
816 ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
817 runtime_ll_cache_stats[cpu].n != 0) {
818 print_ll_cache_misses(cpu, evsel, avg);
819 } else if (perf_evsel__match(evsel, HARDWARE, HW_CACHE_MISSES) &&
820 runtime_cacherefs_stats[cpu].n != 0) {
821 total = avg_stats(&runtime_cacherefs_stats[cpu]);
449 822
450 if (total) 823 if (total)
451 ratio = avg * 100 / total; 824 ratio = avg * 100 / total;
452 825
453 fprintf(stderr, " # %10.3f %% ", ratio); 826 fprintf(stderr, " # %8.3f %% of all cache refs ", ratio);
827
828 } else if (perf_evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_FRONTEND)) {
829 print_stalled_cycles_frontend(cpu, evsel, avg);
830 } else if (perf_evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_BACKEND)) {
831 print_stalled_cycles_backend(cpu, evsel, avg);
832 } else if (perf_evsel__match(evsel, HARDWARE, HW_CPU_CYCLES)) {
833 total = avg_stats(&runtime_nsecs_stats[cpu]);
454 834
835 if (total)
836 ratio = 1.0 * avg / total;
837
838 fprintf(stderr, " # %8.3f GHz ", ratio);
455 } else if (runtime_nsecs_stats[cpu].n != 0) { 839 } else if (runtime_nsecs_stats[cpu].n != 0) {
456 total = avg_stats(&runtime_nsecs_stats[cpu]); 840 total = avg_stats(&runtime_nsecs_stats[cpu]);
457 841
458 if (total) 842 if (total)
459 ratio = 1000.0 * avg / total; 843 ratio = 1000.0 * avg / total;
460 844
461 fprintf(stderr, " # %10.3f M/sec", ratio); 845 fprintf(stderr, " # %8.3f M/sec ", ratio);
846 } else {
847 fprintf(stderr, " ");
462 } 848 }
463} 849}
464 850
@@ -505,8 +891,7 @@ static void print_counter_aggr(struct perf_evsel *counter)
505 avg_enabled = avg_stats(&ps->res_stats[1]); 891 avg_enabled = avg_stats(&ps->res_stats[1]);
506 avg_running = avg_stats(&ps->res_stats[2]); 892 avg_running = avg_stats(&ps->res_stats[2]);
507 893
508 fprintf(stderr, " (scaled from %.2f%%)", 894 fprintf(stderr, " [%5.2f%%]", 100 * avg_running / avg_enabled);
509 100 * avg_running / avg_enabled);
510 } 895 }
511 fprintf(stderr, "\n"); 896 fprintf(stderr, "\n");
512} 897}
@@ -548,10 +933,8 @@ static void print_counter(struct perf_evsel *counter)
548 if (!csv_output) { 933 if (!csv_output) {
549 print_noise(counter, 1.0); 934 print_noise(counter, 1.0);
550 935
551 if (run != ena) { 936 if (run != ena)
552 fprintf(stderr, " (scaled from %.2f%%)", 937 fprintf(stderr, " (%.2f%%)", 100.0 * run / ena);
553 100.0 * run / ena);
554 }
555 } 938 }
556 fputc('\n', stderr); 939 fputc('\n', stderr);
557 } 940 }
@@ -591,13 +974,14 @@ static void print_stat(int argc, const char **argv)
591 } 974 }
592 975
593 if (!csv_output) { 976 if (!csv_output) {
594 fprintf(stderr, "\n"); 977 if (!null_run)
595 fprintf(stderr, " %18.9f seconds time elapsed", 978 fprintf(stderr, "\n");
979 fprintf(stderr, " %17.9f seconds time elapsed",
596 avg_stats(&walltime_nsecs_stats)/1e9); 980 avg_stats(&walltime_nsecs_stats)/1e9);
597 if (run_count > 1) { 981 if (run_count > 1) {
598 fprintf(stderr, " ( +- %7.3f%% )", 982 fprintf(stderr, " ");
599 100*stddev_stats(&walltime_nsecs_stats) / 983 print_noise_pct(stddev_stats(&walltime_nsecs_stats),
600 avg_stats(&walltime_nsecs_stats)); 984 avg_stats(&walltime_nsecs_stats));
601 } 985 }
602 fprintf(stderr, "\n\n"); 986 fprintf(stderr, "\n\n");
603 } 987 }
@@ -659,6 +1043,10 @@ static const struct option options[] = {
659 "repeat command and print average + stddev (max: 100)"), 1043 "repeat command and print average + stddev (max: 100)"),
660 OPT_BOOLEAN('n', "null", &null_run, 1044 OPT_BOOLEAN('n', "null", &null_run,
661 "null run - dont start any counters"), 1045 "null run - dont start any counters"),
1046 OPT_INCR('d', "detailed", &detailed_run,
1047 "detailed run - start a lot of events"),
1048 OPT_BOOLEAN('S', "sync", &sync_run,
1049 "call sync() before starting a run"),
662 OPT_CALLBACK_NOOPT('B', "big-num", NULL, NULL, 1050 OPT_CALLBACK_NOOPT('B', "big-num", NULL, NULL,
663 "print large numbers with thousands\' separators", 1051 "print large numbers with thousands\' separators",
664 stat__set_big_num), 1052 stat__set_big_num),
@@ -674,6 +1062,70 @@ static const struct option options[] = {
674 OPT_END() 1062 OPT_END()
675}; 1063};
676 1064
1065/*
1066 * Add default attributes, if there were no attributes specified or
1067 * if -d/--detailed, -d -d or -d -d -d is used:
1068 */
1069static int add_default_attributes(void)
1070{
1071 struct perf_evsel *pos;
1072 size_t attr_nr = 0;
1073 size_t c;
1074
1075 /* Set attrs if no event is selected and !null_run: */
1076 if (null_run)
1077 return 0;
1078
1079 if (!evsel_list->nr_entries) {
1080 for (c = 0; c < ARRAY_SIZE(default_attrs); c++) {
1081 pos = perf_evsel__new(default_attrs + c, c + attr_nr);
1082 if (pos == NULL)
1083 return -1;
1084 perf_evlist__add(evsel_list, pos);
1085 }
1086 attr_nr += c;
1087 }
1088
1089 /* Detailed events get appended to the event list: */
1090
1091 if (detailed_run < 1)
1092 return 0;
1093
1094 /* Append detailed run extra attributes: */
1095 for (c = 0; c < ARRAY_SIZE(detailed_attrs); c++) {
1096 pos = perf_evsel__new(detailed_attrs + c, c + attr_nr);
1097 if (pos == NULL)
1098 return -1;
1099 perf_evlist__add(evsel_list, pos);
1100 }
1101 attr_nr += c;
1102
1103 if (detailed_run < 2)
1104 return 0;
1105
1106 /* Append very detailed run extra attributes: */
1107 for (c = 0; c < ARRAY_SIZE(very_detailed_attrs); c++) {
1108 pos = perf_evsel__new(very_detailed_attrs + c, c + attr_nr);
1109 if (pos == NULL)
1110 return -1;
1111 perf_evlist__add(evsel_list, pos);
1112 }
1113
1114 if (detailed_run < 3)
1115 return 0;
1116
1117 /* Append very, very detailed run extra attributes: */
1118 for (c = 0; c < ARRAY_SIZE(very_very_detailed_attrs); c++) {
1119 pos = perf_evsel__new(very_very_detailed_attrs + c, c + attr_nr);
1120 if (pos == NULL)
1121 return -1;
1122 perf_evlist__add(evsel_list, pos);
1123 }
1124
1125
1126 return 0;
1127}
1128
677int cmd_stat(int argc, const char **argv, const char *prefix __used) 1129int cmd_stat(int argc, const char **argv, const char *prefix __used)
678{ 1130{
679 struct perf_evsel *pos; 1131 struct perf_evsel *pos;
@@ -719,17 +1171,8 @@ int cmd_stat(int argc, const char **argv, const char *prefix __used)
719 usage_with_options(stat_usage, options); 1171 usage_with_options(stat_usage, options);
720 } 1172 }
721 1173
722 /* Set attrs and nr_counters if no event is selected and !null_run */ 1174 if (add_default_attributes())
723 if (!null_run && !evsel_list->nr_entries) { 1175 goto out;
724 size_t c;
725
726 for (c = 0; c < ARRAY_SIZE(default_attrs); ++c) {
727 pos = perf_evsel__new(&default_attrs[c], c);
728 if (pos == NULL)
729 goto out;
730 perf_evlist__add(evsel_list, pos);
731 }
732 }
733 1176
734 if (target_pid != -1) 1177 if (target_pid != -1)
735 target_tid = target_pid; 1178 target_tid = target_pid;
@@ -773,6 +1216,10 @@ int cmd_stat(int argc, const char **argv, const char *prefix __used)
773 for (run_idx = 0; run_idx < run_count; run_idx++) { 1216 for (run_idx = 0; run_idx < run_count; run_idx++) {
774 if (run_count != 1 && verbose) 1217 if (run_count != 1 && verbose)
775 fprintf(stderr, "[ perf stat: executing run #%d ... ]\n", run_idx + 1); 1218 fprintf(stderr, "[ perf stat: executing run #%d ... ]\n", run_idx + 1);
1219
1220 if (sync_run)
1221 sync();
1222
776 status = run_perf_stat(argc, argv); 1223 status = run_perf_stat(argc, argv);
777 } 1224 }
778 1225
diff --git a/tools/perf/feature-tests.mak b/tools/perf/config/feature-tests.mak
index b041ca67a2cb..6170fd2531b5 100644
--- a/tools/perf/feature-tests.mak
+++ b/tools/perf/config/feature-tests.mak
@@ -79,9 +79,15 @@ endef
79endif 79endif
80 80
81ifndef NO_LIBPYTHON 81ifndef NO_LIBPYTHON
82define SOURCE_PYTHON_VERSION
83#include <Python.h>
84#if PY_VERSION_HEX >= 0x03000000
85 #error
86#endif
87int main(void){}
88endef
82define SOURCE_PYTHON_EMBED 89define SOURCE_PYTHON_EMBED
83#include <Python.h> 90#include <Python.h>
84
85int main(void) 91int main(void)
86{ 92{
87 Py_Initialize(); 93 Py_Initialize();
@@ -120,11 +126,3 @@ int main(void)
120 return 0; 126 return 0;
121} 127}
122endef 128endef
123
124# try-cc
125# Usage: option = $(call try-cc, source-to-build, cc-options)
126try-cc = $(shell sh -c \
127 'TMP="$(OUTPUT)$(TMPOUT).$$$$"; \
128 echo "$(1)" | \
129 $(CC) -x c - $(2) -o "$$TMP" > /dev/null 2>&1 && echo y; \
130 rm -f "$$TMP"')
diff --git a/tools/perf/config/utilities.mak b/tools/perf/config/utilities.mak
new file mode 100644
index 000000000000..8046182a19eb
--- /dev/null
+++ b/tools/perf/config/utilities.mak
@@ -0,0 +1,188 @@
1# This allows us to work with the newline character:
2define newline
3
4
5endef
6newline := $(newline)
7
8# nl-escape
9#
10# Usage: escape = $(call nl-escape[,escape])
11#
12# This is used as the common way to specify
13# what should replace a newline when escaping
14# newlines; the default is a bizarre string.
15#
16nl-escape = $(or $(1),m822df3020w6a44id34bt574ctac44eb9f4n)
17
18# escape-nl
19#
20# Usage: escaped-text = $(call escape-nl,text[,escape])
21#
22# GNU make's $(shell ...) function converts to a
23# single space each newline character in the output
24# produced during the expansion; this may not be
25# desirable.
26#
27# The only solution is to change each newline into
28# something that won't be converted, so that the
29# information can be recovered later with
30# $(call unescape-nl...)
31#
32escape-nl = $(subst $(newline),$(call nl-escape,$(2)),$(1))
33
34# unescape-nl
35#
36# Usage: text = $(call unescape-nl,escaped-text[,escape])
37#
38# See escape-nl.
39#
40unescape-nl = $(subst $(call nl-escape,$(2)),$(newline),$(1))
41
42# shell-escape-nl
43#
44# Usage: $(shell some-command | $(call shell-escape-nl[,escape]))
45#
46# Use this to escape newlines from within a shell call;
47# the default escape is a bizarre string.
48#
49# NOTE: The escape is used directly as a string constant
50# in an `awk' program that is delimited by shell
51# single-quotes, so be wary of the characters
52# that are chosen.
53#
54define shell-escape-nl
55awk 'NR==1 {t=$$0} NR>1 {t=t "$(nl-escape)" $$0} END {printf t}'
56endef
57
58# shell-unescape-nl
59#
60# Usage: $(shell some-command | $(call shell-unescape-nl[,escape]))
61#
62# Use this to unescape newlines from within a shell call;
63# the default escape is a bizarre string.
64#
65# NOTE: The escape is used directly as an extended regular
66# expression constant in an `awk' program that is
67# delimited by shell single-quotes, so be wary
68# of the characters that are chosen.
69#
70# (The bash shell has a bug where `{gsub(...),...}' is
71# misinterpreted as a brace expansion; this can be
72# overcome by putting a space between `{' and `gsub').
73#
74define shell-unescape-nl
75awk 'NR==1 {t=$$0} NR>1 {t=t "\n" $$0} END { gsub(/$(nl-escape)/,"\n",t); printf t }'
76endef
77
78# escape-for-shell-sq
79#
80# Usage: embeddable-text = $(call escape-for-shell-sq,text)
81#
82# This function produces text that is suitable for
83# embedding in a shell string that is delimited by
84# single-quotes.
85#
86escape-for-shell-sq = $(subst ','\'',$(1))
87
88# shell-sq
89#
90# Usage: single-quoted-and-escaped-text = $(call shell-sq,text)
91#
92shell-sq = '$(escape-for-shell-sq)'
93
94# shell-wordify
95#
96# Usage: wordified-text = $(call shell-wordify,text)
97#
98# For instance:
99#
100# |define text
101# |hello
102# |world
103# |endef
104# |
105# |target:
106# | echo $(call shell-wordify,$(text))
107#
108# At least GNU make gets confused by expanding a newline
109# within the context of a command line of a makefile rule
110# (this is in constrast to a `$(shell ...)' function call,
111# which can handle it just fine).
112#
113# This function avoids the problem by producing a string
114# that works as a shell word, regardless of whether or
115# not it contains a newline.
116#
117# If the text to be wordified contains a newline, then
118# an intrictate shell command substitution is constructed
119# to render the text as a single line; when the shell
120# processes the resulting escaped text, it transforms
121# it into the original unescaped text.
122#
123# If the text does not contain a newline, then this function
124# produces the same results as the `$(shell-sq)' function.
125#
126shell-wordify = $(if $(findstring $(newline),$(1)),$(_sw-esc-nl),$(shell-sq))
127define _sw-esc-nl
128"$$(echo $(call escape-nl,$(shell-sq),$(2)) | $(call shell-unescape-nl,$(2)))"
129endef
130
131# is-absolute
132#
133# Usage: bool-value = $(call is-absolute,path)
134#
135is-absolute = $(shell echo $(shell-sq) | grep ^/ -q && echo y)
136
137# lookup
138#
139# Usage: absolute-executable-path-or-empty = $(call lookup,path)
140#
141# (It's necessary to use `sh -c' because GNU make messes up by
142# trying too hard and getting things wrong).
143#
144lookup = $(call unescape-nl,$(shell sh -c $(_l-sh)))
145_l-sh = $(call shell-sq,command -v $(shell-sq) | $(call shell-escape-nl,))
146
147# is-executable
148#
149# Usage: bool-value = $(call is-executable,path)
150#
151# (It's necessary to use `sh -c' because GNU make messes up by
152# trying too hard and getting things wrong).
153#
154is-executable = $(call _is-executable-helper,$(shell-sq))
155_is-executable-helper = $(shell sh -c $(_is-executable-sh))
156_is-executable-sh = $(call shell-sq,test -f $(1) -a -x $(1) && echo y)
157
158# get-executable
159#
160# Usage: absolute-executable-path-or-empty = $(call get-executable,path)
161#
162# The goal is to get an absolute path for an executable;
163# the `command -v' is defined by POSIX, but it's not
164# necessarily very portable, so it's only used if
165# relative path resolution is requested, as determined
166# by the presence of a leading `/'.
167#
168get-executable = $(if $(1),$(if $(is-absolute),$(_ge-abspath),$(lookup)))
169_ge-abspath = $(if $(is-executable),$(1))
170
171# get-supplied-or-default-executable
172#
173# Usage: absolute-executable-path-or-empty = $(call get-executable-or-default,variable,default)
174#
175define get-executable-or-default
176$(if $($(1)),$(call _ge_attempt,$($(1)),$(1)),$(call _ge_attempt,$(2)))
177endef
178_ge_attempt = $(or $(get-executable),$(_gea_warn),$(call _gea_err,$(2)))
179_gea_warn = $(warning The path '$(1)' is not executable.)
180_gea_err = $(if $(1),$(error Please set '$(1)' appropriately))
181
182# try-cc
183# Usage: option = $(call try-cc, source-to-build, cc-options)
184try-cc = $(shell sh -c \
185 'TMP="$(OUTPUT)$(TMPOUT).$$$$"; \
186 echo "$(1)" | \
187 $(CC) -x c - $(2) -o "$$TMP" > /dev/null 2>&1 && echo y; \
188 rm -f "$$TMP"')
diff --git a/tools/perf/util/include/asm/alternative-asm.h b/tools/perf/util/include/asm/alternative-asm.h
new file mode 100644
index 000000000000..6789d788d494
--- /dev/null
+++ b/tools/perf/util/include/asm/alternative-asm.h
@@ -0,0 +1,8 @@
1#ifndef _PERF_ASM_ALTERNATIVE_ASM_H
2#define _PERF_ASM_ALTERNATIVE_ASM_H
3
4/* Just disable it so we can build arch/x86/lib/memcpy_64.S for perf bench: */
5
6#define altinstruction_entry #
7
8#endif
diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c
index 952b4ae3d954..41982c373faf 100644
--- a/tools/perf/util/parse-events.c
+++ b/tools/perf/util/parse-events.c
@@ -31,34 +31,36 @@ char debugfs_path[MAXPATHLEN];
31#define CSW(x) .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_##x 31#define CSW(x) .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_##x
32 32
33static struct event_symbol event_symbols[] = { 33static struct event_symbol event_symbols[] = {
34 { CHW(CPU_CYCLES), "cpu-cycles", "cycles" }, 34 { CHW(CPU_CYCLES), "cpu-cycles", "cycles" },
35 { CHW(INSTRUCTIONS), "instructions", "" }, 35 { CHW(STALLED_CYCLES_FRONTEND), "stalled-cycles-frontend", "idle-cycles-frontend" },
36 { CHW(CACHE_REFERENCES), "cache-references", "" }, 36 { CHW(STALLED_CYCLES_BACKEND), "stalled-cycles-backend", "idle-cycles-backend" },
37 { CHW(CACHE_MISSES), "cache-misses", "" }, 37 { CHW(INSTRUCTIONS), "instructions", "" },
38 { CHW(BRANCH_INSTRUCTIONS), "branch-instructions", "branches" }, 38 { CHW(CACHE_REFERENCES), "cache-references", "" },
39 { CHW(BRANCH_MISSES), "branch-misses", "" }, 39 { CHW(CACHE_MISSES), "cache-misses", "" },
40 { CHW(BUS_CYCLES), "bus-cycles", "" }, 40 { CHW(BRANCH_INSTRUCTIONS), "branch-instructions", "branches" },
41 41 { CHW(BRANCH_MISSES), "branch-misses", "" },
42 { CSW(CPU_CLOCK), "cpu-clock", "" }, 42 { CHW(BUS_CYCLES), "bus-cycles", "" },
43 { CSW(TASK_CLOCK), "task-clock", "" }, 43
44 { CSW(PAGE_FAULTS), "page-faults", "faults" }, 44 { CSW(CPU_CLOCK), "cpu-clock", "" },
45 { CSW(PAGE_FAULTS_MIN), "minor-faults", "" }, 45 { CSW(TASK_CLOCK), "task-clock", "" },
46 { CSW(PAGE_FAULTS_MAJ), "major-faults", "" }, 46 { CSW(PAGE_FAULTS), "page-faults", "faults" },
47 { CSW(CONTEXT_SWITCHES), "context-switches", "cs" }, 47 { CSW(PAGE_FAULTS_MIN), "minor-faults", "" },
48 { CSW(CPU_MIGRATIONS), "cpu-migrations", "migrations" }, 48 { CSW(PAGE_FAULTS_MAJ), "major-faults", "" },
49 { CSW(ALIGNMENT_FAULTS), "alignment-faults", "" }, 49 { CSW(CONTEXT_SWITCHES), "context-switches", "cs" },
50 { CSW(EMULATION_FAULTS), "emulation-faults", "" }, 50 { CSW(CPU_MIGRATIONS), "cpu-migrations", "migrations" },
51 { CSW(ALIGNMENT_FAULTS), "alignment-faults", "" },
52 { CSW(EMULATION_FAULTS), "emulation-faults", "" },
51}; 53};
52 54
53#define __PERF_EVENT_FIELD(config, name) \ 55#define __PERF_EVENT_FIELD(config, name) \
54 ((config & PERF_EVENT_##name##_MASK) >> PERF_EVENT_##name##_SHIFT) 56 ((config & PERF_EVENT_##name##_MASK) >> PERF_EVENT_##name##_SHIFT)
55 57
56#define PERF_EVENT_RAW(config) __PERF_EVENT_FIELD(config, RAW) 58#define PERF_EVENT_RAW(config) __PERF_EVENT_FIELD(config, RAW)
57#define PERF_EVENT_CONFIG(config) __PERF_EVENT_FIELD(config, CONFIG) 59#define PERF_EVENT_CONFIG(config) __PERF_EVENT_FIELD(config, CONFIG)
58#define PERF_EVENT_TYPE(config) __PERF_EVENT_FIELD(config, TYPE) 60#define PERF_EVENT_TYPE(config) __PERF_EVENT_FIELD(config, TYPE)
59#define PERF_EVENT_ID(config) __PERF_EVENT_FIELD(config, EVENT) 61#define PERF_EVENT_ID(config) __PERF_EVENT_FIELD(config, EVENT)
60 62
61static const char *hw_event_names[] = { 63static const char *hw_event_names[PERF_COUNT_HW_MAX] = {
62 "cycles", 64 "cycles",
63 "instructions", 65 "instructions",
64 "cache-references", 66 "cache-references",
@@ -66,11 +68,13 @@ static const char *hw_event_names[] = {
66 "branches", 68 "branches",
67 "branch-misses", 69 "branch-misses",
68 "bus-cycles", 70 "bus-cycles",
71 "stalled-cycles-frontend",
72 "stalled-cycles-backend",
69}; 73};
70 74
71static const char *sw_event_names[] = { 75static const char *sw_event_names[PERF_COUNT_SW_MAX] = {
72 "cpu-clock-msecs", 76 "cpu-clock",
73 "task-clock-msecs", 77 "task-clock",
74 "page-faults", 78 "page-faults",
75 "context-switches", 79 "context-switches",
76 "CPU-migrations", 80 "CPU-migrations",
@@ -307,7 +311,7 @@ const char *__event_name(int type, u64 config)
307 311
308 switch (type) { 312 switch (type) {
309 case PERF_TYPE_HARDWARE: 313 case PERF_TYPE_HARDWARE:
310 if (config < PERF_COUNT_HW_MAX) 314 if (config < PERF_COUNT_HW_MAX && hw_event_names[config])
311 return hw_event_names[config]; 315 return hw_event_names[config];
312 return "unknown-hardware"; 316 return "unknown-hardware";
313 317
@@ -333,7 +337,7 @@ const char *__event_name(int type, u64 config)
333 } 337 }
334 338
335 case PERF_TYPE_SOFTWARE: 339 case PERF_TYPE_SOFTWARE:
336 if (config < PERF_COUNT_SW_MAX) 340 if (config < PERF_COUNT_SW_MAX && sw_event_names[config])
337 return sw_event_names[config]; 341 return sw_event_names[config];
338 return "unknown-software"; 342 return "unknown-software";
339 343
@@ -648,13 +652,15 @@ static int check_events(const char *str, unsigned int i)
648 int n; 652 int n;
649 653
650 n = strlen(event_symbols[i].symbol); 654 n = strlen(event_symbols[i].symbol);
651 if (!strncmp(str, event_symbols[i].symbol, n)) 655 if (!strncasecmp(str, event_symbols[i].symbol, n))
652 return n; 656 return n;
653 657
654 n = strlen(event_symbols[i].alias); 658 n = strlen(event_symbols[i].alias);
655 if (n) 659 if (n) {
656 if (!strncmp(str, event_symbols[i].alias, n)) 660 if (!strncasecmp(str, event_symbols[i].alias, n))
657 return n; 661 return n;
662 }
663
658 return 0; 664 return 0;
659} 665}
660 666
@@ -718,15 +724,22 @@ parse_numeric_event(const char **strp, struct perf_event_attr *attr)
718 return EVT_FAILED; 724 return EVT_FAILED;
719} 725}
720 726
721static enum event_result 727static int
722parse_event_modifier(const char **strp, struct perf_event_attr *attr) 728parse_event_modifier(const char **strp, struct perf_event_attr *attr)
723{ 729{
724 const char *str = *strp; 730 const char *str = *strp;
725 int exclude = 0; 731 int exclude = 0;
726 int eu = 0, ek = 0, eh = 0, precise = 0; 732 int eu = 0, ek = 0, eh = 0, precise = 0;
727 733
728 if (*str++ != ':') 734 if (!*str)
735 return 0;
736
737 if (*str == ',')
729 return 0; 738 return 0;
739
740 if (*str++ != ':')
741 return -1;
742
730 while (*str) { 743 while (*str) {
731 if (*str == 'u') { 744 if (*str == 'u') {
732 if (!exclude) 745 if (!exclude)
@@ -747,14 +760,16 @@ parse_event_modifier(const char **strp, struct perf_event_attr *attr)
747 760
748 ++str; 761 ++str;
749 } 762 }
750 if (str >= *strp + 2) { 763 if (str < *strp + 2)
751 *strp = str; 764 return -1;
752 attr->exclude_user = eu; 765
753 attr->exclude_kernel = ek; 766 *strp = str;
754 attr->exclude_hv = eh; 767
755 attr->precise_ip = precise; 768 attr->exclude_user = eu;
756 return 1; 769 attr->exclude_kernel = ek;
757 } 770 attr->exclude_hv = eh;
771 attr->precise_ip = precise;
772
758 return 0; 773 return 0;
759} 774}
760 775
@@ -797,7 +812,12 @@ parse_event_symbols(const struct option *opt, const char **str,
797 return EVT_FAILED; 812 return EVT_FAILED;
798 813
799modifier: 814modifier:
800 parse_event_modifier(str, attr); 815 if (parse_event_modifier(str, attr) < 0) {
816 fprintf(stderr, "invalid event modifier: '%s'\n", *str);
817 fprintf(stderr, "Run 'perf list' for a list of valid events and modifiers\n");
818
819 return EVT_FAILED;
820 }
801 821
802 return ret; 822 return ret;
803} 823}
@@ -912,7 +932,7 @@ void print_tracepoint_events(const char *subsys_glob, const char *event_glob)
912 932
913 snprintf(evt_path, MAXPATHLEN, "%s:%s", 933 snprintf(evt_path, MAXPATHLEN, "%s:%s",
914 sys_dirent.d_name, evt_dirent.d_name); 934 sys_dirent.d_name, evt_dirent.d_name);
915 printf(" %-42s [%s]\n", evt_path, 935 printf(" %-50s [%s]\n", evt_path,
916 event_type_descriptors[PERF_TYPE_TRACEPOINT]); 936 event_type_descriptors[PERF_TYPE_TRACEPOINT]);
917 } 937 }
918 closedir(evt_dir); 938 closedir(evt_dir);
@@ -977,7 +997,7 @@ void print_events_type(u8 type)
977 else 997 else
978 snprintf(name, sizeof(name), "%s", syms->symbol); 998 snprintf(name, sizeof(name), "%s", syms->symbol);
979 999
980 printf(" %-42s [%s]\n", name, 1000 printf(" %-50s [%s]\n", name,
981 event_type_descriptors[type]); 1001 event_type_descriptors[type]);
982 } 1002 }
983} 1003}
@@ -995,11 +1015,10 @@ int print_hwcache_events(const char *event_glob)
995 for (i = 0; i < PERF_COUNT_HW_CACHE_RESULT_MAX; i++) { 1015 for (i = 0; i < PERF_COUNT_HW_CACHE_RESULT_MAX; i++) {
996 char *name = event_cache_name(type, op, i); 1016 char *name = event_cache_name(type, op, i);
997 1017
998 if (event_glob != NULL && 1018 if (event_glob != NULL && !strglobmatch(name, event_glob))
999 !strglobmatch(name, event_glob))
1000 continue; 1019 continue;
1001 1020
1002 printf(" %-42s [%s]\n", name, 1021 printf(" %-50s [%s]\n", name,
1003 event_type_descriptors[PERF_TYPE_HW_CACHE]); 1022 event_type_descriptors[PERF_TYPE_HW_CACHE]);
1004 ++printed; 1023 ++printed;
1005 } 1024 }
@@ -1009,14 +1028,16 @@ int print_hwcache_events(const char *event_glob)
1009 return printed; 1028 return printed;
1010} 1029}
1011 1030
1031#define MAX_NAME_LEN 100
1032
1012/* 1033/*
1013 * Print the help text for the event symbols: 1034 * Print the help text for the event symbols:
1014 */ 1035 */
1015void print_events(const char *event_glob) 1036void print_events(const char *event_glob)
1016{ 1037{
1017 struct event_symbol *syms = event_symbols;
1018 unsigned int i, type, prev_type = -1, printed = 0, ntypes_printed = 0; 1038 unsigned int i, type, prev_type = -1, printed = 0, ntypes_printed = 0;
1019 char name[40]; 1039 struct event_symbol *syms = event_symbols;
1040 char name[MAX_NAME_LEN];
1020 1041
1021 printf("\n"); 1042 printf("\n");
1022 printf("List of pre-defined events (to be used in -e):\n"); 1043 printf("List of pre-defined events (to be used in -e):\n");
@@ -1036,10 +1057,10 @@ void print_events(const char *event_glob)
1036 continue; 1057 continue;
1037 1058
1038 if (strlen(syms->alias)) 1059 if (strlen(syms->alias))
1039 sprintf(name, "%s OR %s", syms->symbol, syms->alias); 1060 snprintf(name, MAX_NAME_LEN, "%s OR %s", syms->symbol, syms->alias);
1040 else 1061 else
1041 strcpy(name, syms->symbol); 1062 strncpy(name, syms->symbol, MAX_NAME_LEN);
1042 printf(" %-42s [%s]\n", name, 1063 printf(" %-50s [%s]\n", name,
1043 event_type_descriptors[type]); 1064 event_type_descriptors[type]);
1044 1065
1045 prev_type = type; 1066 prev_type = type;
@@ -1056,12 +1077,12 @@ void print_events(const char *event_glob)
1056 return; 1077 return;
1057 1078
1058 printf("\n"); 1079 printf("\n");
1059 printf(" %-42s [%s]\n", 1080 printf(" %-50s [%s]\n",
1060 "rNNN (see 'perf list --help' on how to encode it)", 1081 "rNNN (see 'perf list --help' on how to encode it)",
1061 event_type_descriptors[PERF_TYPE_RAW]); 1082 event_type_descriptors[PERF_TYPE_RAW]);
1062 printf("\n"); 1083 printf("\n");
1063 1084
1064 printf(" %-42s [%s]\n", 1085 printf(" %-50s [%s]\n",
1065 "mem:<addr>[:access]", 1086 "mem:<addr>[:access]",
1066 event_type_descriptors[PERF_TYPE_BREAKPOINT]); 1087 event_type_descriptors[PERF_TYPE_BREAKPOINT]);
1067 printf("\n"); 1088 printf("\n");
diff --git a/tools/perf/util/probe-finder.c b/tools/perf/util/probe-finder.c
index b7c85ce466a1..3b9d0b800d5c 100644
--- a/tools/perf/util/probe-finder.c
+++ b/tools/perf/util/probe-finder.c
@@ -1471,6 +1471,38 @@ static int find_probe_point_by_func(struct probe_finder *pf)
1471 return _param.retval; 1471 return _param.retval;
1472} 1472}
1473 1473
1474struct pubname_callback_param {
1475 char *function;
1476 char *file;
1477 Dwarf_Die *cu_die;
1478 Dwarf_Die *sp_die;
1479 int found;
1480};
1481
1482static int pubname_search_cb(Dwarf *dbg, Dwarf_Global *gl, void *data)
1483{
1484 struct pubname_callback_param *param = data;
1485
1486 if (dwarf_offdie(dbg, gl->die_offset, param->sp_die)) {
1487 if (dwarf_tag(param->sp_die) != DW_TAG_subprogram)
1488 return DWARF_CB_OK;
1489
1490 if (die_compare_name(param->sp_die, param->function)) {
1491 if (!dwarf_offdie(dbg, gl->cu_offset, param->cu_die))
1492 return DWARF_CB_OK;
1493
1494 if (param->file &&
1495 strtailcmp(param->file, dwarf_decl_file(param->sp_die)))
1496 return DWARF_CB_OK;
1497
1498 param->found = 1;
1499 return DWARF_CB_ABORT;
1500 }
1501 }
1502
1503 return DWARF_CB_OK;
1504}
1505
1474/* Find probe points from debuginfo */ 1506/* Find probe points from debuginfo */
1475static int find_probes(int fd, struct probe_finder *pf) 1507static int find_probes(int fd, struct probe_finder *pf)
1476{ 1508{
@@ -1498,6 +1530,28 @@ static int find_probes(int fd, struct probe_finder *pf)
1498 1530
1499 off = 0; 1531 off = 0;
1500 line_list__init(&pf->lcache); 1532 line_list__init(&pf->lcache);
1533
1534 /* Fastpath: lookup by function name from .debug_pubnames section */
1535 if (pp->function) {
1536 struct pubname_callback_param pubname_param = {
1537 .function = pp->function,
1538 .file = pp->file,
1539 .cu_die = &pf->cu_die,
1540 .sp_die = &pf->sp_die,
1541 .found = 0,
1542 };
1543 struct dwarf_callback_param probe_param = {
1544 .data = pf,
1545 };
1546
1547 dwarf_getpubnames(dbg, pubname_search_cb, &pubname_param, 0);
1548 if (pubname_param.found) {
1549 ret = probe_point_search_cb(&pf->sp_die, &probe_param);
1550 if (ret)
1551 goto found;
1552 }
1553 }
1554
1501 /* Loop on CUs (Compilation Unit) */ 1555 /* Loop on CUs (Compilation Unit) */
1502 while (!dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL)) { 1556 while (!dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL)) {
1503 /* Get the DIE(Debugging Information Entry) of this CU */ 1557 /* Get the DIE(Debugging Information Entry) of this CU */
@@ -1525,6 +1579,8 @@ static int find_probes(int fd, struct probe_finder *pf)
1525 } 1579 }
1526 off = noff; 1580 off = noff;
1527 } 1581 }
1582
1583found:
1528 line_list__free(&pf->lcache); 1584 line_list__free(&pf->lcache);
1529 if (dwfl) 1585 if (dwfl)
1530 dwfl_end(dwfl); 1586 dwfl_end(dwfl);
@@ -1946,6 +2002,22 @@ int find_line_range(int fd, struct line_range *lr)
1946 return -EBADF; 2002 return -EBADF;
1947 } 2003 }
1948 2004
2005 /* Fastpath: lookup by function name from .debug_pubnames section */
2006 if (lr->function) {
2007 struct pubname_callback_param pubname_param = {
2008 .function = lr->function, .file = lr->file,
2009 .cu_die = &lf.cu_die, .sp_die = &lf.sp_die, .found = 0};
2010 struct dwarf_callback_param line_range_param = {
2011 .data = (void *)&lf, .retval = 0};
2012
2013 dwarf_getpubnames(dbg, pubname_search_cb, &pubname_param, 0);
2014 if (pubname_param.found) {
2015 line_range_search_cb(&lf.sp_die, &line_range_param);
2016 if (lf.found)
2017 goto found;
2018 }
2019 }
2020
1949 /* Loop on CUs (Compilation Unit) */ 2021 /* Loop on CUs (Compilation Unit) */
1950 while (!lf.found && ret >= 0) { 2022 while (!lf.found && ret >= 0) {
1951 if (dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL) != 0) 2023 if (dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL) != 0)
@@ -1974,6 +2046,7 @@ int find_line_range(int fd, struct line_range *lr)
1974 off = noff; 2046 off = noff;
1975 } 2047 }
1976 2048
2049found:
1977 /* Store comp_dir */ 2050 /* Store comp_dir */
1978 if (lf.found) { 2051 if (lf.found) {
1979 comp_dir = cu_get_comp_dir(&lf.cu_die); 2052 comp_dir = cu_get_comp_dir(&lf.cu_die);
diff --git a/tools/perf/util/probe-finder.h b/tools/perf/util/probe-finder.h
index beaefc3c1223..605730a366db 100644
--- a/tools/perf/util/probe-finder.h
+++ b/tools/perf/util/probe-finder.h
@@ -49,6 +49,7 @@ struct probe_finder {
49 Dwarf_Addr addr; /* Address */ 49 Dwarf_Addr addr; /* Address */
50 const char *fname; /* Real file name */ 50 const char *fname; /* Real file name */
51 Dwarf_Die cu_die; /* Current CU */ 51 Dwarf_Die cu_die; /* Current CU */
52 Dwarf_Die sp_die;
52 struct list_head lcache; /* Line cache for lazy match */ 53 struct list_head lcache; /* Line cache for lazy match */
53 54
54 /* For variable searching */ 55 /* For variable searching */
@@ -83,6 +84,7 @@ struct line_finder {
83 int lno_s; /* Start line number */ 84 int lno_s; /* Start line number */
84 int lno_e; /* End line number */ 85 int lno_e; /* End line number */
85 Dwarf_Die cu_die; /* Current CU */ 86 Dwarf_Die cu_die; /* Current CU */
87 Dwarf_Die sp_die;
86 int found; 88 int found;
87}; 89};
88 90
diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c
index 99c722672f84..b5c7d818001c 100644
--- a/tools/perf/util/python.c
+++ b/tools/perf/util/python.c
@@ -810,6 +810,9 @@ static struct {
810 { "COUNT_HW_CACHE_RESULT_ACCESS", PERF_COUNT_HW_CACHE_RESULT_ACCESS }, 810 { "COUNT_HW_CACHE_RESULT_ACCESS", PERF_COUNT_HW_CACHE_RESULT_ACCESS },
811 { "COUNT_HW_CACHE_RESULT_MISS", PERF_COUNT_HW_CACHE_RESULT_MISS }, 811 { "COUNT_HW_CACHE_RESULT_MISS", PERF_COUNT_HW_CACHE_RESULT_MISS },
812 812
813 { "COUNT_HW_STALLED_CYCLES_FRONTEND", PERF_COUNT_HW_STALLED_CYCLES_FRONTEND },
814 { "COUNT_HW_STALLED_CYCLES_BACKEND", PERF_COUNT_HW_STALLED_CYCLES_BACKEND },
815
813 { "COUNT_SW_CPU_CLOCK", PERF_COUNT_SW_CPU_CLOCK }, 816 { "COUNT_SW_CPU_CLOCK", PERF_COUNT_SW_CPU_CLOCK },
814 { "COUNT_SW_TASK_CLOCK", PERF_COUNT_SW_TASK_CLOCK }, 817 { "COUNT_SW_TASK_CLOCK", PERF_COUNT_SW_TASK_CLOCK },
815 { "COUNT_SW_PAGE_FAULTS", PERF_COUNT_SW_PAGE_FAULTS }, 818 { "COUNT_SW_PAGE_FAULTS", PERF_COUNT_SW_PAGE_FAULTS },
diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c
index caa224522fea..fff66741f18d 100644
--- a/tools/perf/util/session.c
+++ b/tools/perf/util/session.c
@@ -1156,6 +1156,18 @@ size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp)
1156 return ret; 1156 return ret;
1157} 1157}
1158 1158
1159struct perf_evsel *perf_session__find_first_evtype(struct perf_session *session,
1160 unsigned int type)
1161{
1162 struct perf_evsel *pos;
1163
1164 list_for_each_entry(pos, &session->evlist->entries, node) {
1165 if (pos->attr.type == type)
1166 return pos;
1167 }
1168 return NULL;
1169}
1170
1159void perf_session__print_symbols(union perf_event *event, 1171void perf_session__print_symbols(union perf_event *event,
1160 struct perf_sample *sample, 1172 struct perf_sample *sample,
1161 struct perf_session *session) 1173 struct perf_session *session)
diff --git a/tools/perf/util/session.h b/tools/perf/util/session.h
index 1ac481fc1100..8daaa2d15396 100644
--- a/tools/perf/util/session.h
+++ b/tools/perf/util/session.h
@@ -162,6 +162,9 @@ static inline int perf_session__parse_sample(struct perf_session *session,
162 session->sample_id_all, sample); 162 session->sample_id_all, sample);
163} 163}
164 164
165struct perf_evsel *perf_session__find_first_evtype(struct perf_session *session,
166 unsigned int type);
167
165void perf_session__print_symbols(union perf_event *event, 168void perf_session__print_symbols(union perf_event *event,
166 struct perf_sample *sample, 169 struct perf_sample *sample,
167 struct perf_session *session); 170 struct perf_session *session);
diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c
index f06c10f092ba..516876dfbe52 100644
--- a/tools/perf/util/symbol.c
+++ b/tools/perf/util/symbol.c
@@ -31,13 +31,13 @@
31#define NT_GNU_BUILD_ID 3 31#define NT_GNU_BUILD_ID 3
32#endif 32#endif
33 33
34static bool dso__build_id_equal(const struct dso *self, u8 *build_id); 34static bool dso__build_id_equal(const struct dso *dso, u8 *build_id);
35static int elf_read_build_id(Elf *elf, void *bf, size_t size); 35static int elf_read_build_id(Elf *elf, void *bf, size_t size);
36static void dsos__add(struct list_head *head, struct dso *dso); 36static void dsos__add(struct list_head *head, struct dso *dso);
37static struct map *map__new2(u64 start, struct dso *dso, enum map_type type); 37static struct map *map__new2(u64 start, struct dso *dso, enum map_type type);
38static int dso__load_kernel_sym(struct dso *self, struct map *map, 38static int dso__load_kernel_sym(struct dso *dso, struct map *map,
39 symbol_filter_t filter); 39 symbol_filter_t filter);
40static int dso__load_guest_kernel_sym(struct dso *self, struct map *map, 40static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map,
41 symbol_filter_t filter); 41 symbol_filter_t filter);
42static int vmlinux_path__nr_entries; 42static int vmlinux_path__nr_entries;
43static char **vmlinux_path; 43static char **vmlinux_path;
@@ -49,27 +49,27 @@ struct symbol_conf symbol_conf = {
49 .symfs = "", 49 .symfs = "",
50}; 50};
51 51
52int dso__name_len(const struct dso *self) 52int dso__name_len(const struct dso *dso)
53{ 53{
54 if (verbose) 54 if (verbose)
55 return self->long_name_len; 55 return dso->long_name_len;
56 56
57 return self->short_name_len; 57 return dso->short_name_len;
58} 58}
59 59
60bool dso__loaded(const struct dso *self, enum map_type type) 60bool dso__loaded(const struct dso *dso, enum map_type type)
61{ 61{
62 return self->loaded & (1 << type); 62 return dso->loaded & (1 << type);
63} 63}
64 64
65bool dso__sorted_by_name(const struct dso *self, enum map_type type) 65bool dso__sorted_by_name(const struct dso *dso, enum map_type type)
66{ 66{
67 return self->sorted_by_name & (1 << type); 67 return dso->sorted_by_name & (1 << type);
68} 68}
69 69
70static void dso__set_sorted_by_name(struct dso *self, enum map_type type) 70static void dso__set_sorted_by_name(struct dso *dso, enum map_type type)
71{ 71{
72 self->sorted_by_name |= (1 << type); 72 dso->sorted_by_name |= (1 << type);
73} 73}
74 74
75bool symbol_type__is_a(char symbol_type, enum map_type map_type) 75bool symbol_type__is_a(char symbol_type, enum map_type map_type)
@@ -84,9 +84,9 @@ bool symbol_type__is_a(char symbol_type, enum map_type map_type)
84 } 84 }
85} 85}
86 86
87static void symbols__fixup_end(struct rb_root *self) 87static void symbols__fixup_end(struct rb_root *symbols)
88{ 88{
89 struct rb_node *nd, *prevnd = rb_first(self); 89 struct rb_node *nd, *prevnd = rb_first(symbols);
90 struct symbol *curr, *prev; 90 struct symbol *curr, *prev;
91 91
92 if (prevnd == NULL) 92 if (prevnd == NULL)
@@ -107,10 +107,10 @@ static void symbols__fixup_end(struct rb_root *self)
107 curr->end = roundup(curr->start, 4096); 107 curr->end = roundup(curr->start, 4096);
108} 108}
109 109
110static void __map_groups__fixup_end(struct map_groups *self, enum map_type type) 110static void __map_groups__fixup_end(struct map_groups *mg, enum map_type type)
111{ 111{
112 struct map *prev, *curr; 112 struct map *prev, *curr;
113 struct rb_node *nd, *prevnd = rb_first(&self->maps[type]); 113 struct rb_node *nd, *prevnd = rb_first(&mg->maps[type]);
114 114
115 if (prevnd == NULL) 115 if (prevnd == NULL)
116 return; 116 return;
@@ -130,128 +130,128 @@ static void __map_groups__fixup_end(struct map_groups *self, enum map_type type)
130 curr->end = ~0ULL; 130 curr->end = ~0ULL;
131} 131}
132 132
133static void map_groups__fixup_end(struct map_groups *self) 133static void map_groups__fixup_end(struct map_groups *mg)
134{ 134{
135 int i; 135 int i;
136 for (i = 0; i < MAP__NR_TYPES; ++i) 136 for (i = 0; i < MAP__NR_TYPES; ++i)
137 __map_groups__fixup_end(self, i); 137 __map_groups__fixup_end(mg, i);
138} 138}
139 139
140static struct symbol *symbol__new(u64 start, u64 len, u8 binding, 140static struct symbol *symbol__new(u64 start, u64 len, u8 binding,
141 const char *name) 141 const char *name)
142{ 142{
143 size_t namelen = strlen(name) + 1; 143 size_t namelen = strlen(name) + 1;
144 struct symbol *self = calloc(1, (symbol_conf.priv_size + 144 struct symbol *sym = calloc(1, (symbol_conf.priv_size +
145 sizeof(*self) + namelen)); 145 sizeof(*sym) + namelen));
146 if (self == NULL) 146 if (sym == NULL)
147 return NULL; 147 return NULL;
148 148
149 if (symbol_conf.priv_size) 149 if (symbol_conf.priv_size)
150 self = ((void *)self) + symbol_conf.priv_size; 150 sym = ((void *)sym) + symbol_conf.priv_size;
151
152 self->start = start;
153 self->end = len ? start + len - 1 : start;
154 self->binding = binding;
155 self->namelen = namelen - 1;
156 151
157 pr_debug4("%s: %s %#" PRIx64 "-%#" PRIx64 "\n", __func__, name, start, self->end); 152 sym->start = start;
153 sym->end = len ? start + len - 1 : start;
154 sym->binding = binding;
155 sym->namelen = namelen - 1;
158 156
159 memcpy(self->name, name, namelen); 157 pr_debug4("%s: %s %#" PRIx64 "-%#" PRIx64 "\n",
158 __func__, name, start, sym->end);
159 memcpy(sym->name, name, namelen);
160 160
161 return self; 161 return sym;
162} 162}
163 163
164void symbol__delete(struct symbol *self) 164void symbol__delete(struct symbol *sym)
165{ 165{
166 free(((void *)self) - symbol_conf.priv_size); 166 free(((void *)sym) - symbol_conf.priv_size);
167} 167}
168 168
169static size_t symbol__fprintf(struct symbol *self, FILE *fp) 169static size_t symbol__fprintf(struct symbol *sym, FILE *fp)
170{ 170{
171 return fprintf(fp, " %" PRIx64 "-%" PRIx64 " %c %s\n", 171 return fprintf(fp, " %" PRIx64 "-%" PRIx64 " %c %s\n",
172 self->start, self->end, 172 sym->start, sym->end,
173 self->binding == STB_GLOBAL ? 'g' : 173 sym->binding == STB_GLOBAL ? 'g' :
174 self->binding == STB_LOCAL ? 'l' : 'w', 174 sym->binding == STB_LOCAL ? 'l' : 'w',
175 self->name); 175 sym->name);
176} 176}
177 177
178void dso__set_long_name(struct dso *self, char *name) 178void dso__set_long_name(struct dso *dso, char *name)
179{ 179{
180 if (name == NULL) 180 if (name == NULL)
181 return; 181 return;
182 self->long_name = name; 182 dso->long_name = name;
183 self->long_name_len = strlen(name); 183 dso->long_name_len = strlen(name);
184} 184}
185 185
186static void dso__set_short_name(struct dso *self, const char *name) 186static void dso__set_short_name(struct dso *dso, const char *name)
187{ 187{
188 if (name == NULL) 188 if (name == NULL)
189 return; 189 return;
190 self->short_name = name; 190 dso->short_name = name;
191 self->short_name_len = strlen(name); 191 dso->short_name_len = strlen(name);
192} 192}
193 193
194static void dso__set_basename(struct dso *self) 194static void dso__set_basename(struct dso *dso)
195{ 195{
196 dso__set_short_name(self, basename(self->long_name)); 196 dso__set_short_name(dso, basename(dso->long_name));
197} 197}
198 198
199struct dso *dso__new(const char *name) 199struct dso *dso__new(const char *name)
200{ 200{
201 struct dso *self = calloc(1, sizeof(*self) + strlen(name) + 1); 201 struct dso *dso = calloc(1, sizeof(*dso) + strlen(name) + 1);
202 202
203 if (self != NULL) { 203 if (dso != NULL) {
204 int i; 204 int i;
205 strcpy(self->name, name); 205 strcpy(dso->name, name);
206 dso__set_long_name(self, self->name); 206 dso__set_long_name(dso, dso->name);
207 dso__set_short_name(self, self->name); 207 dso__set_short_name(dso, dso->name);
208 for (i = 0; i < MAP__NR_TYPES; ++i) 208 for (i = 0; i < MAP__NR_TYPES; ++i)
209 self->symbols[i] = self->symbol_names[i] = RB_ROOT; 209 dso->symbols[i] = dso->symbol_names[i] = RB_ROOT;
210 self->symtab_type = SYMTAB__NOT_FOUND; 210 dso->symtab_type = SYMTAB__NOT_FOUND;
211 self->loaded = 0; 211 dso->loaded = 0;
212 self->sorted_by_name = 0; 212 dso->sorted_by_name = 0;
213 self->has_build_id = 0; 213 dso->has_build_id = 0;
214 self->kernel = DSO_TYPE_USER; 214 dso->kernel = DSO_TYPE_USER;
215 INIT_LIST_HEAD(&self->node); 215 INIT_LIST_HEAD(&dso->node);
216 } 216 }
217 217
218 return self; 218 return dso;
219} 219}
220 220
221static void symbols__delete(struct rb_root *self) 221static void symbols__delete(struct rb_root *symbols)
222{ 222{
223 struct symbol *pos; 223 struct symbol *pos;
224 struct rb_node *next = rb_first(self); 224 struct rb_node *next = rb_first(symbols);
225 225
226 while (next) { 226 while (next) {
227 pos = rb_entry(next, struct symbol, rb_node); 227 pos = rb_entry(next, struct symbol, rb_node);
228 next = rb_next(&pos->rb_node); 228 next = rb_next(&pos->rb_node);
229 rb_erase(&pos->rb_node, self); 229 rb_erase(&pos->rb_node, symbols);
230 symbol__delete(pos); 230 symbol__delete(pos);
231 } 231 }
232} 232}
233 233
234void dso__delete(struct dso *self) 234void dso__delete(struct dso *dso)
235{ 235{
236 int i; 236 int i;
237 for (i = 0; i < MAP__NR_TYPES; ++i) 237 for (i = 0; i < MAP__NR_TYPES; ++i)
238 symbols__delete(&self->symbols[i]); 238 symbols__delete(&dso->symbols[i]);
239 if (self->sname_alloc) 239 if (dso->sname_alloc)
240 free((char *)self->short_name); 240 free((char *)dso->short_name);
241 if (self->lname_alloc) 241 if (dso->lname_alloc)
242 free(self->long_name); 242 free(dso->long_name);
243 free(self); 243 free(dso);
244} 244}
245 245
246void dso__set_build_id(struct dso *self, void *build_id) 246void dso__set_build_id(struct dso *dso, void *build_id)
247{ 247{
248 memcpy(self->build_id, build_id, sizeof(self->build_id)); 248 memcpy(dso->build_id, build_id, sizeof(dso->build_id));
249 self->has_build_id = 1; 249 dso->has_build_id = 1;
250} 250}
251 251
252static void symbols__insert(struct rb_root *self, struct symbol *sym) 252static void symbols__insert(struct rb_root *symbols, struct symbol *sym)
253{ 253{
254 struct rb_node **p = &self->rb_node; 254 struct rb_node **p = &symbols->rb_node;
255 struct rb_node *parent = NULL; 255 struct rb_node *parent = NULL;
256 const u64 ip = sym->start; 256 const u64 ip = sym->start;
257 struct symbol *s; 257 struct symbol *s;
@@ -265,17 +265,17 @@ static void symbols__insert(struct rb_root *self, struct symbol *sym)
265 p = &(*p)->rb_right; 265 p = &(*p)->rb_right;
266 } 266 }
267 rb_link_node(&sym->rb_node, parent, p); 267 rb_link_node(&sym->rb_node, parent, p);
268 rb_insert_color(&sym->rb_node, self); 268 rb_insert_color(&sym->rb_node, symbols);
269} 269}
270 270
271static struct symbol *symbols__find(struct rb_root *self, u64 ip) 271static struct symbol *symbols__find(struct rb_root *symbols, u64 ip)
272{ 272{
273 struct rb_node *n; 273 struct rb_node *n;
274 274
275 if (self == NULL) 275 if (symbols == NULL)
276 return NULL; 276 return NULL;
277 277
278 n = self->rb_node; 278 n = symbols->rb_node;
279 279
280 while (n) { 280 while (n) {
281 struct symbol *s = rb_entry(n, struct symbol, rb_node); 281 struct symbol *s = rb_entry(n, struct symbol, rb_node);
@@ -296,9 +296,9 @@ struct symbol_name_rb_node {
296 struct symbol sym; 296 struct symbol sym;
297}; 297};
298 298
299static void symbols__insert_by_name(struct rb_root *self, struct symbol *sym) 299static void symbols__insert_by_name(struct rb_root *symbols, struct symbol *sym)
300{ 300{
301 struct rb_node **p = &self->rb_node; 301 struct rb_node **p = &symbols->rb_node;
302 struct rb_node *parent = NULL; 302 struct rb_node *parent = NULL;
303 struct symbol_name_rb_node *symn, *s; 303 struct symbol_name_rb_node *symn, *s;
304 304
@@ -313,27 +313,29 @@ static void symbols__insert_by_name(struct rb_root *self, struct symbol *sym)
313 p = &(*p)->rb_right; 313 p = &(*p)->rb_right;
314 } 314 }
315 rb_link_node(&symn->rb_node, parent, p); 315 rb_link_node(&symn->rb_node, parent, p);
316 rb_insert_color(&symn->rb_node, self); 316 rb_insert_color(&symn->rb_node, symbols);
317} 317}
318 318
319static void symbols__sort_by_name(struct rb_root *self, struct rb_root *source) 319static void symbols__sort_by_name(struct rb_root *symbols,
320 struct rb_root *source)
320{ 321{
321 struct rb_node *nd; 322 struct rb_node *nd;
322 323
323 for (nd = rb_first(source); nd; nd = rb_next(nd)) { 324 for (nd = rb_first(source); nd; nd = rb_next(nd)) {
324 struct symbol *pos = rb_entry(nd, struct symbol, rb_node); 325 struct symbol *pos = rb_entry(nd, struct symbol, rb_node);
325 symbols__insert_by_name(self, pos); 326 symbols__insert_by_name(symbols, pos);
326 } 327 }
327} 328}
328 329
329static struct symbol *symbols__find_by_name(struct rb_root *self, const char *name) 330static struct symbol *symbols__find_by_name(struct rb_root *symbols,
331 const char *name)
330{ 332{
331 struct rb_node *n; 333 struct rb_node *n;
332 334
333 if (self == NULL) 335 if (symbols == NULL)
334 return NULL; 336 return NULL;
335 337
336 n = self->rb_node; 338 n = symbols->rb_node;
337 339
338 while (n) { 340 while (n) {
339 struct symbol_name_rb_node *s; 341 struct symbol_name_rb_node *s;
@@ -353,29 +355,29 @@ static struct symbol *symbols__find_by_name(struct rb_root *self, const char *na
353 return NULL; 355 return NULL;
354} 356}
355 357
356struct symbol *dso__find_symbol(struct dso *self, 358struct symbol *dso__find_symbol(struct dso *dso,
357 enum map_type type, u64 addr) 359 enum map_type type, u64 addr)
358{ 360{
359 return symbols__find(&self->symbols[type], addr); 361 return symbols__find(&dso->symbols[type], addr);
360} 362}
361 363
362struct symbol *dso__find_symbol_by_name(struct dso *self, enum map_type type, 364struct symbol *dso__find_symbol_by_name(struct dso *dso, enum map_type type,
363 const char *name) 365 const char *name)
364{ 366{
365 return symbols__find_by_name(&self->symbol_names[type], name); 367 return symbols__find_by_name(&dso->symbol_names[type], name);
366} 368}
367 369
368void dso__sort_by_name(struct dso *self, enum map_type type) 370void dso__sort_by_name(struct dso *dso, enum map_type type)
369{ 371{
370 dso__set_sorted_by_name(self, type); 372 dso__set_sorted_by_name(dso, type);
371 return symbols__sort_by_name(&self->symbol_names[type], 373 return symbols__sort_by_name(&dso->symbol_names[type],
372 &self->symbols[type]); 374 &dso->symbols[type]);
373} 375}
374 376
375int build_id__sprintf(const u8 *self, int len, char *bf) 377int build_id__sprintf(const u8 *build_id, int len, char *bf)
376{ 378{
377 char *bid = bf; 379 char *bid = bf;
378 const u8 *raw = self; 380 const u8 *raw = build_id;
379 int i; 381 int i;
380 382
381 for (i = 0; i < len; ++i) { 383 for (i = 0; i < len; ++i) {
@@ -384,24 +386,25 @@ int build_id__sprintf(const u8 *self, int len, char *bf)
384 bid += 2; 386 bid += 2;
385 } 387 }
386 388
387 return raw - self; 389 return raw - build_id;
388} 390}
389 391
390size_t dso__fprintf_buildid(struct dso *self, FILE *fp) 392size_t dso__fprintf_buildid(struct dso *dso, FILE *fp)
391{ 393{
392 char sbuild_id[BUILD_ID_SIZE * 2 + 1]; 394 char sbuild_id[BUILD_ID_SIZE * 2 + 1];
393 395
394 build_id__sprintf(self->build_id, sizeof(self->build_id), sbuild_id); 396 build_id__sprintf(dso->build_id, sizeof(dso->build_id), sbuild_id);
395 return fprintf(fp, "%s", sbuild_id); 397 return fprintf(fp, "%s", sbuild_id);
396} 398}
397 399
398size_t dso__fprintf_symbols_by_name(struct dso *self, enum map_type type, FILE *fp) 400size_t dso__fprintf_symbols_by_name(struct dso *dso,
401 enum map_type type, FILE *fp)
399{ 402{
400 size_t ret = 0; 403 size_t ret = 0;
401 struct rb_node *nd; 404 struct rb_node *nd;
402 struct symbol_name_rb_node *pos; 405 struct symbol_name_rb_node *pos;
403 406
404 for (nd = rb_first(&self->symbol_names[type]); nd; nd = rb_next(nd)) { 407 for (nd = rb_first(&dso->symbol_names[type]); nd; nd = rb_next(nd)) {
405 pos = rb_entry(nd, struct symbol_name_rb_node, rb_node); 408 pos = rb_entry(nd, struct symbol_name_rb_node, rb_node);
406 fprintf(fp, "%s\n", pos->sym.name); 409 fprintf(fp, "%s\n", pos->sym.name);
407 } 410 }
@@ -409,18 +412,18 @@ size_t dso__fprintf_symbols_by_name(struct dso *self, enum map_type type, FILE *
409 return ret; 412 return ret;
410} 413}
411 414
412size_t dso__fprintf(struct dso *self, enum map_type type, FILE *fp) 415size_t dso__fprintf(struct dso *dso, enum map_type type, FILE *fp)
413{ 416{
414 struct rb_node *nd; 417 struct rb_node *nd;
415 size_t ret = fprintf(fp, "dso: %s (", self->short_name); 418 size_t ret = fprintf(fp, "dso: %s (", dso->short_name);
416 419
417 if (self->short_name != self->long_name) 420 if (dso->short_name != dso->long_name)
418 ret += fprintf(fp, "%s, ", self->long_name); 421 ret += fprintf(fp, "%s, ", dso->long_name);
419 ret += fprintf(fp, "%s, %sloaded, ", map_type__name[type], 422 ret += fprintf(fp, "%s, %sloaded, ", map_type__name[type],
420 self->loaded ? "" : "NOT "); 423 dso->loaded ? "" : "NOT ");
421 ret += dso__fprintf_buildid(self, fp); 424 ret += dso__fprintf_buildid(dso, fp);
422 ret += fprintf(fp, ")\n"); 425 ret += fprintf(fp, ")\n");
423 for (nd = rb_first(&self->symbols[type]); nd; nd = rb_next(nd)) { 426 for (nd = rb_first(&dso->symbols[type]); nd; nd = rb_next(nd)) {
424 struct symbol *pos = rb_entry(nd, struct symbol, rb_node); 427 struct symbol *pos = rb_entry(nd, struct symbol, rb_node);
425 ret += symbol__fprintf(pos, fp); 428 ret += symbol__fprintf(pos, fp);
426 } 429 }
@@ -543,10 +546,10 @@ static int map__process_kallsym_symbol(void *arg, const char *name,
543 * so that we can in the next step set the symbol ->end address and then 546 * so that we can in the next step set the symbol ->end address and then
544 * call kernel_maps__split_kallsyms. 547 * call kernel_maps__split_kallsyms.
545 */ 548 */
546static int dso__load_all_kallsyms(struct dso *self, const char *filename, 549static int dso__load_all_kallsyms(struct dso *dso, const char *filename,
547 struct map *map) 550 struct map *map)
548{ 551{
549 struct process_kallsyms_args args = { .map = map, .dso = self, }; 552 struct process_kallsyms_args args = { .map = map, .dso = dso, };
550 return kallsyms__parse(filename, &args, map__process_kallsym_symbol); 553 return kallsyms__parse(filename, &args, map__process_kallsym_symbol);
551} 554}
552 555
@@ -555,7 +558,7 @@ static int dso__load_all_kallsyms(struct dso *self, const char *filename,
555 * kernel range is broken in several maps, named [kernel].N, as we don't have 558 * kernel range is broken in several maps, named [kernel].N, as we don't have
556 * the original ELF section names vmlinux have. 559 * the original ELF section names vmlinux have.
557 */ 560 */
558static int dso__split_kallsyms(struct dso *self, struct map *map, 561static int dso__split_kallsyms(struct dso *dso, struct map *map,
559 symbol_filter_t filter) 562 symbol_filter_t filter)
560{ 563{
561 struct map_groups *kmaps = map__kmap(map)->kmaps; 564 struct map_groups *kmaps = map__kmap(map)->kmaps;
@@ -563,7 +566,7 @@ static int dso__split_kallsyms(struct dso *self, struct map *map,
563 struct map *curr_map = map; 566 struct map *curr_map = map;
564 struct symbol *pos; 567 struct symbol *pos;
565 int count = 0, moved = 0; 568 int count = 0, moved = 0;
566 struct rb_root *root = &self->symbols[map->type]; 569 struct rb_root *root = &dso->symbols[map->type];
567 struct rb_node *next = rb_first(root); 570 struct rb_node *next = rb_first(root);
568 int kernel_range = 0; 571 int kernel_range = 0;
569 572
@@ -582,7 +585,7 @@ static int dso__split_kallsyms(struct dso *self, struct map *map,
582 585
583 if (strcmp(curr_map->dso->short_name, module)) { 586 if (strcmp(curr_map->dso->short_name, module)) {
584 if (curr_map != map && 587 if (curr_map != map &&
585 self->kernel == DSO_TYPE_GUEST_KERNEL && 588 dso->kernel == DSO_TYPE_GUEST_KERNEL &&
586 machine__is_default_guest(machine)) { 589 machine__is_default_guest(machine)) {
587 /* 590 /*
588 * We assume all symbols of a module are 591 * We assume all symbols of a module are
@@ -618,14 +621,14 @@ static int dso__split_kallsyms(struct dso *self, struct map *map,
618 pos->end = curr_map->map_ip(curr_map, pos->end); 621 pos->end = curr_map->map_ip(curr_map, pos->end);
619 } else if (curr_map != map) { 622 } else if (curr_map != map) {
620 char dso_name[PATH_MAX]; 623 char dso_name[PATH_MAX];
621 struct dso *dso; 624 struct dso *ndso;
622 625
623 if (count == 0) { 626 if (count == 0) {
624 curr_map = map; 627 curr_map = map;
625 goto filter_symbol; 628 goto filter_symbol;
626 } 629 }
627 630
628 if (self->kernel == DSO_TYPE_GUEST_KERNEL) 631 if (dso->kernel == DSO_TYPE_GUEST_KERNEL)
629 snprintf(dso_name, sizeof(dso_name), 632 snprintf(dso_name, sizeof(dso_name),
630 "[guest.kernel].%d", 633 "[guest.kernel].%d",
631 kernel_range++); 634 kernel_range++);
@@ -634,15 +637,15 @@ static int dso__split_kallsyms(struct dso *self, struct map *map,
634 "[kernel].%d", 637 "[kernel].%d",
635 kernel_range++); 638 kernel_range++);
636 639
637 dso = dso__new(dso_name); 640 ndso = dso__new(dso_name);
638 if (dso == NULL) 641 if (ndso == NULL)
639 return -1; 642 return -1;
640 643
641 dso->kernel = self->kernel; 644 ndso->kernel = dso->kernel;
642 645
643 curr_map = map__new2(pos->start, dso, map->type); 646 curr_map = map__new2(pos->start, ndso, map->type);
644 if (curr_map == NULL) { 647 if (curr_map == NULL) {
645 dso__delete(dso); 648 dso__delete(ndso);
646 return -1; 649 return -1;
647 } 650 }
648 651
@@ -665,7 +668,7 @@ discard_symbol: rb_erase(&pos->rb_node, root);
665 } 668 }
666 669
667 if (curr_map != map && 670 if (curr_map != map &&
668 self->kernel == DSO_TYPE_GUEST_KERNEL && 671 dso->kernel == DSO_TYPE_GUEST_KERNEL &&
669 machine__is_default_guest(kmaps->machine)) { 672 machine__is_default_guest(kmaps->machine)) {
670 dso__set_loaded(curr_map->dso, curr_map->type); 673 dso__set_loaded(curr_map->dso, curr_map->type);
671 } 674 }
@@ -673,21 +676,21 @@ discard_symbol: rb_erase(&pos->rb_node, root);
673 return count + moved; 676 return count + moved;
674} 677}
675 678
676int dso__load_kallsyms(struct dso *self, const char *filename, 679int dso__load_kallsyms(struct dso *dso, const char *filename,
677 struct map *map, symbol_filter_t filter) 680 struct map *map, symbol_filter_t filter)
678{ 681{
679 if (dso__load_all_kallsyms(self, filename, map) < 0) 682 if (dso__load_all_kallsyms(dso, filename, map) < 0)
680 return -1; 683 return -1;
681 684
682 if (self->kernel == DSO_TYPE_GUEST_KERNEL) 685 if (dso->kernel == DSO_TYPE_GUEST_KERNEL)
683 self->symtab_type = SYMTAB__GUEST_KALLSYMS; 686 dso->symtab_type = SYMTAB__GUEST_KALLSYMS;
684 else 687 else
685 self->symtab_type = SYMTAB__KALLSYMS; 688 dso->symtab_type = SYMTAB__KALLSYMS;
686 689
687 return dso__split_kallsyms(self, map, filter); 690 return dso__split_kallsyms(dso, map, filter);
688} 691}
689 692
690static int dso__load_perf_map(struct dso *self, struct map *map, 693static int dso__load_perf_map(struct dso *dso, struct map *map,
691 symbol_filter_t filter) 694 symbol_filter_t filter)
692{ 695{
693 char *line = NULL; 696 char *line = NULL;
@@ -695,7 +698,7 @@ static int dso__load_perf_map(struct dso *self, struct map *map,
695 FILE *file; 698 FILE *file;
696 int nr_syms = 0; 699 int nr_syms = 0;
697 700
698 file = fopen(self->long_name, "r"); 701 file = fopen(dso->long_name, "r");
699 if (file == NULL) 702 if (file == NULL)
700 goto out_failure; 703 goto out_failure;
701 704
@@ -733,7 +736,7 @@ static int dso__load_perf_map(struct dso *self, struct map *map,
733 if (filter && filter(map, sym)) 736 if (filter && filter(map, sym))
734 symbol__delete(sym); 737 symbol__delete(sym);
735 else { 738 else {
736 symbols__insert(&self->symbols[map->type], sym); 739 symbols__insert(&dso->symbols[map->type], sym);
737 nr_syms++; 740 nr_syms++;
738 } 741 }
739 } 742 }
@@ -752,7 +755,7 @@ out_failure:
752/** 755/**
753 * elf_symtab__for_each_symbol - iterate thru all the symbols 756 * elf_symtab__for_each_symbol - iterate thru all the symbols
754 * 757 *
755 * @self: struct elf_symtab instance to iterate 758 * @syms: struct elf_symtab instance to iterate
756 * @idx: uint32_t idx 759 * @idx: uint32_t idx
757 * @sym: GElf_Sym iterator 760 * @sym: GElf_Sym iterator
758 */ 761 */
@@ -852,7 +855,7 @@ static Elf_Scn *elf_section_by_name(Elf *elf, GElf_Ehdr *ep,
852 * And always look at the original dso, not at debuginfo packages, that 855 * And always look at the original dso, not at debuginfo packages, that
853 * have the PLT data stripped out (shdr_rel_plt.sh_type == SHT_NOBITS). 856 * have the PLT data stripped out (shdr_rel_plt.sh_type == SHT_NOBITS).
854 */ 857 */
855static int dso__synthesize_plt_symbols(struct dso *self, struct map *map, 858static int dso__synthesize_plt_symbols(struct dso *dso, struct map *map,
856 symbol_filter_t filter) 859 symbol_filter_t filter)
857{ 860{
858 uint32_t nr_rel_entries, idx; 861 uint32_t nr_rel_entries, idx;
@@ -871,7 +874,7 @@ static int dso__synthesize_plt_symbols(struct dso *self, struct map *map,
871 char name[PATH_MAX]; 874 char name[PATH_MAX];
872 875
873 snprintf(name, sizeof(name), "%s%s", 876 snprintf(name, sizeof(name), "%s%s",
874 symbol_conf.symfs, self->long_name); 877 symbol_conf.symfs, dso->long_name);
875 fd = open(name, O_RDONLY); 878 fd = open(name, O_RDONLY);
876 if (fd < 0) 879 if (fd < 0)
877 goto out; 880 goto out;
@@ -947,7 +950,7 @@ static int dso__synthesize_plt_symbols(struct dso *self, struct map *map,
947 if (filter && filter(map, f)) 950 if (filter && filter(map, f))
948 symbol__delete(f); 951 symbol__delete(f);
949 else { 952 else {
950 symbols__insert(&self->symbols[map->type], f); 953 symbols__insert(&dso->symbols[map->type], f);
951 ++nr; 954 ++nr;
952 } 955 }
953 } 956 }
@@ -969,7 +972,7 @@ static int dso__synthesize_plt_symbols(struct dso *self, struct map *map,
969 if (filter && filter(map, f)) 972 if (filter && filter(map, f))
970 symbol__delete(f); 973 symbol__delete(f);
971 else { 974 else {
972 symbols__insert(&self->symbols[map->type], f); 975 symbols__insert(&dso->symbols[map->type], f);
973 ++nr; 976 ++nr;
974 } 977 }
975 } 978 }
@@ -985,29 +988,30 @@ out_close:
985 return nr; 988 return nr;
986out: 989out:
987 pr_debug("%s: problems reading %s PLT info.\n", 990 pr_debug("%s: problems reading %s PLT info.\n",
988 __func__, self->long_name); 991 __func__, dso->long_name);
989 return 0; 992 return 0;
990} 993}
991 994
992static bool elf_sym__is_a(GElf_Sym *self, enum map_type type) 995static bool elf_sym__is_a(GElf_Sym *sym, enum map_type type)
993{ 996{
994 switch (type) { 997 switch (type) {
995 case MAP__FUNCTION: 998 case MAP__FUNCTION:
996 return elf_sym__is_function(self); 999 return elf_sym__is_function(sym);
997 case MAP__VARIABLE: 1000 case MAP__VARIABLE:
998 return elf_sym__is_object(self); 1001 return elf_sym__is_object(sym);
999 default: 1002 default:
1000 return false; 1003 return false;
1001 } 1004 }
1002} 1005}
1003 1006
1004static bool elf_sec__is_a(GElf_Shdr *self, Elf_Data *secstrs, enum map_type type) 1007static bool elf_sec__is_a(GElf_Shdr *shdr, Elf_Data *secstrs,
1008 enum map_type type)
1005{ 1009{
1006 switch (type) { 1010 switch (type) {
1007 case MAP__FUNCTION: 1011 case MAP__FUNCTION:
1008 return elf_sec__is_text(self, secstrs); 1012 return elf_sec__is_text(shdr, secstrs);
1009 case MAP__VARIABLE: 1013 case MAP__VARIABLE:
1010 return elf_sec__is_data(self, secstrs); 1014 return elf_sec__is_data(shdr, secstrs);
1011 default: 1015 default:
1012 return false; 1016 return false;
1013 } 1017 }
@@ -1032,13 +1036,13 @@ static size_t elf_addr_to_index(Elf *elf, GElf_Addr addr)
1032 return -1; 1036 return -1;
1033} 1037}
1034 1038
1035static int dso__load_sym(struct dso *self, struct map *map, const char *name, 1039static int dso__load_sym(struct dso *dso, struct map *map, const char *name,
1036 int fd, symbol_filter_t filter, int kmodule, 1040 int fd, symbol_filter_t filter, int kmodule,
1037 int want_symtab) 1041 int want_symtab)
1038{ 1042{
1039 struct kmap *kmap = self->kernel ? map__kmap(map) : NULL; 1043 struct kmap *kmap = dso->kernel ? map__kmap(map) : NULL;
1040 struct map *curr_map = map; 1044 struct map *curr_map = map;
1041 struct dso *curr_dso = self; 1045 struct dso *curr_dso = dso;
1042 Elf_Data *symstrs, *secstrs; 1046 Elf_Data *symstrs, *secstrs;
1043 uint32_t nr_syms; 1047 uint32_t nr_syms;
1044 int err = -1; 1048 int err = -1;
@@ -1064,14 +1068,14 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name,
1064 } 1068 }
1065 1069
1066 /* Always reject images with a mismatched build-id: */ 1070 /* Always reject images with a mismatched build-id: */
1067 if (self->has_build_id) { 1071 if (dso->has_build_id) {
1068 u8 build_id[BUILD_ID_SIZE]; 1072 u8 build_id[BUILD_ID_SIZE];
1069 1073
1070 if (elf_read_build_id(elf, build_id, 1074 if (elf_read_build_id(elf, build_id,
1071 BUILD_ID_SIZE) != BUILD_ID_SIZE) 1075 BUILD_ID_SIZE) != BUILD_ID_SIZE)
1072 goto out_elf_end; 1076 goto out_elf_end;
1073 1077
1074 if (!dso__build_id_equal(self, build_id)) 1078 if (!dso__build_id_equal(dso, build_id))
1075 goto out_elf_end; 1079 goto out_elf_end;
1076 } 1080 }
1077 1081
@@ -1112,13 +1116,14 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name,
1112 nr_syms = shdr.sh_size / shdr.sh_entsize; 1116 nr_syms = shdr.sh_size / shdr.sh_entsize;
1113 1117
1114 memset(&sym, 0, sizeof(sym)); 1118 memset(&sym, 0, sizeof(sym));
1115 if (self->kernel == DSO_TYPE_USER) { 1119 if (dso->kernel == DSO_TYPE_USER) {
1116 self->adjust_symbols = (ehdr.e_type == ET_EXEC || 1120 dso->adjust_symbols = (ehdr.e_type == ET_EXEC ||
1117 elf_section_by_name(elf, &ehdr, &shdr, 1121 elf_section_by_name(elf, &ehdr, &shdr,
1118 ".gnu.prelink_undo", 1122 ".gnu.prelink_undo",
1119 NULL) != NULL); 1123 NULL) != NULL);
1120 } else self->adjust_symbols = 0; 1124 } else {
1121 1125 dso->adjust_symbols = 0;
1126 }
1122 elf_symtab__for_each_symbol(syms, nr_syms, idx, sym) { 1127 elf_symtab__for_each_symbol(syms, nr_syms, idx, sym) {
1123 struct symbol *f; 1128 struct symbol *f;
1124 const char *elf_name = elf_sym__name(&sym, symstrs); 1129 const char *elf_name = elf_sym__name(&sym, symstrs);
@@ -1168,22 +1173,22 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name,
1168 (sym.st_value & 1)) 1173 (sym.st_value & 1))
1169 --sym.st_value; 1174 --sym.st_value;
1170 1175
1171 if (self->kernel != DSO_TYPE_USER || kmodule) { 1176 if (dso->kernel != DSO_TYPE_USER || kmodule) {
1172 char dso_name[PATH_MAX]; 1177 char dso_name[PATH_MAX];
1173 1178
1174 if (strcmp(section_name, 1179 if (strcmp(section_name,
1175 (curr_dso->short_name + 1180 (curr_dso->short_name +
1176 self->short_name_len)) == 0) 1181 dso->short_name_len)) == 0)
1177 goto new_symbol; 1182 goto new_symbol;
1178 1183
1179 if (strcmp(section_name, ".text") == 0) { 1184 if (strcmp(section_name, ".text") == 0) {
1180 curr_map = map; 1185 curr_map = map;
1181 curr_dso = self; 1186 curr_dso = dso;
1182 goto new_symbol; 1187 goto new_symbol;
1183 } 1188 }
1184 1189
1185 snprintf(dso_name, sizeof(dso_name), 1190 snprintf(dso_name, sizeof(dso_name),
1186 "%s%s", self->short_name, section_name); 1191 "%s%s", dso->short_name, section_name);
1187 1192
1188 curr_map = map_groups__find_by_name(kmap->kmaps, map->type, dso_name); 1193 curr_map = map_groups__find_by_name(kmap->kmaps, map->type, dso_name);
1189 if (curr_map == NULL) { 1194 if (curr_map == NULL) {
@@ -1195,9 +1200,9 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name,
1195 curr_dso = dso__new(dso_name); 1200 curr_dso = dso__new(dso_name);
1196 if (curr_dso == NULL) 1201 if (curr_dso == NULL)
1197 goto out_elf_end; 1202 goto out_elf_end;
1198 curr_dso->kernel = self->kernel; 1203 curr_dso->kernel = dso->kernel;
1199 curr_dso->long_name = self->long_name; 1204 curr_dso->long_name = dso->long_name;
1200 curr_dso->long_name_len = self->long_name_len; 1205 curr_dso->long_name_len = dso->long_name_len;
1201 curr_map = map__new2(start, curr_dso, 1206 curr_map = map__new2(start, curr_dso,
1202 map->type); 1207 map->type);
1203 if (curr_map == NULL) { 1208 if (curr_map == NULL) {
@@ -1206,9 +1211,9 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name,
1206 } 1211 }
1207 curr_map->map_ip = identity__map_ip; 1212 curr_map->map_ip = identity__map_ip;
1208 curr_map->unmap_ip = identity__map_ip; 1213 curr_map->unmap_ip = identity__map_ip;
1209 curr_dso->symtab_type = self->symtab_type; 1214 curr_dso->symtab_type = dso->symtab_type;
1210 map_groups__insert(kmap->kmaps, curr_map); 1215 map_groups__insert(kmap->kmaps, curr_map);
1211 dsos__add(&self->node, curr_dso); 1216 dsos__add(&dso->node, curr_dso);
1212 dso__set_loaded(curr_dso, map->type); 1217 dso__set_loaded(curr_dso, map->type);
1213 } else 1218 } else
1214 curr_dso = curr_map->dso; 1219 curr_dso = curr_map->dso;
@@ -1250,7 +1255,7 @@ new_symbol:
1250 * For misannotated, zeroed, ASM function sizes. 1255 * For misannotated, zeroed, ASM function sizes.
1251 */ 1256 */
1252 if (nr > 0) { 1257 if (nr > 0) {
1253 symbols__fixup_end(&self->symbols[map->type]); 1258 symbols__fixup_end(&dso->symbols[map->type]);
1254 if (kmap) { 1259 if (kmap) {
1255 /* 1260 /*
1256 * We need to fixup this here too because we create new 1261 * We need to fixup this here too because we create new
@@ -1266,9 +1271,9 @@ out_close:
1266 return err; 1271 return err;
1267} 1272}
1268 1273
1269static bool dso__build_id_equal(const struct dso *self, u8 *build_id) 1274static bool dso__build_id_equal(const struct dso *dso, u8 *build_id)
1270{ 1275{
1271 return memcmp(self->build_id, build_id, sizeof(self->build_id)) == 0; 1276 return memcmp(dso->build_id, build_id, sizeof(dso->build_id)) == 0;
1272} 1277}
1273 1278
1274bool __dsos__read_build_ids(struct list_head *head, bool with_hits) 1279bool __dsos__read_build_ids(struct list_head *head, bool with_hits)
@@ -1429,7 +1434,7 @@ out:
1429 return err; 1434 return err;
1430} 1435}
1431 1436
1432char dso__symtab_origin(const struct dso *self) 1437char dso__symtab_origin(const struct dso *dso)
1433{ 1438{
1434 static const char origin[] = { 1439 static const char origin[] = {
1435 [SYMTAB__KALLSYMS] = 'k', 1440 [SYMTAB__KALLSYMS] = 'k',
@@ -1444,12 +1449,12 @@ char dso__symtab_origin(const struct dso *self)
1444 [SYMTAB__GUEST_KMODULE] = 'G', 1449 [SYMTAB__GUEST_KMODULE] = 'G',
1445 }; 1450 };
1446 1451
1447 if (self == NULL || self->symtab_type == SYMTAB__NOT_FOUND) 1452 if (dso == NULL || dso->symtab_type == SYMTAB__NOT_FOUND)
1448 return '!'; 1453 return '!';
1449 return origin[self->symtab_type]; 1454 return origin[dso->symtab_type];
1450} 1455}
1451 1456
1452int dso__load(struct dso *self, struct map *map, symbol_filter_t filter) 1457int dso__load(struct dso *dso, struct map *map, symbol_filter_t filter)
1453{ 1458{
1454 int size = PATH_MAX; 1459 int size = PATH_MAX;
1455 char *name; 1460 char *name;
@@ -1459,12 +1464,12 @@ int dso__load(struct dso *self, struct map *map, symbol_filter_t filter)
1459 const char *root_dir; 1464 const char *root_dir;
1460 int want_symtab; 1465 int want_symtab;
1461 1466
1462 dso__set_loaded(self, map->type); 1467 dso__set_loaded(dso, map->type);
1463 1468
1464 if (self->kernel == DSO_TYPE_KERNEL) 1469 if (dso->kernel == DSO_TYPE_KERNEL)
1465 return dso__load_kernel_sym(self, map, filter); 1470 return dso__load_kernel_sym(dso, map, filter);
1466 else if (self->kernel == DSO_TYPE_GUEST_KERNEL) 1471 else if (dso->kernel == DSO_TYPE_GUEST_KERNEL)
1467 return dso__load_guest_kernel_sym(self, map, filter); 1472 return dso__load_guest_kernel_sym(dso, map, filter);
1468 1473
1469 if (map->groups && map->groups->machine) 1474 if (map->groups && map->groups->machine)
1470 machine = map->groups->machine; 1475 machine = map->groups->machine;
@@ -1475,11 +1480,11 @@ int dso__load(struct dso *self, struct map *map, symbol_filter_t filter)
1475 if (!name) 1480 if (!name)
1476 return -1; 1481 return -1;
1477 1482
1478 self->adjust_symbols = 0; 1483 dso->adjust_symbols = 0;
1479 1484
1480 if (strncmp(self->name, "/tmp/perf-", 10) == 0) { 1485 if (strncmp(dso->name, "/tmp/perf-", 10) == 0) {
1481 ret = dso__load_perf_map(self, map, filter); 1486 ret = dso__load_perf_map(dso, map, filter);
1482 self->symtab_type = ret > 0 ? SYMTAB__JAVA_JIT : 1487 dso->symtab_type = ret > 0 ? SYMTAB__JAVA_JIT :
1483 SYMTAB__NOT_FOUND; 1488 SYMTAB__NOT_FOUND;
1484 return ret; 1489 return ret;
1485 } 1490 }
@@ -1490,33 +1495,33 @@ int dso__load(struct dso *self, struct map *map, symbol_filter_t filter)
1490 */ 1495 */
1491 want_symtab = 1; 1496 want_symtab = 1;
1492restart: 1497restart:
1493 for (self->symtab_type = SYMTAB__BUILD_ID_CACHE; 1498 for (dso->symtab_type = SYMTAB__BUILD_ID_CACHE;
1494 self->symtab_type != SYMTAB__NOT_FOUND; 1499 dso->symtab_type != SYMTAB__NOT_FOUND;
1495 self->symtab_type++) { 1500 dso->symtab_type++) {
1496 switch (self->symtab_type) { 1501 switch (dso->symtab_type) {
1497 case SYMTAB__BUILD_ID_CACHE: 1502 case SYMTAB__BUILD_ID_CACHE:
1498 /* skip the locally configured cache if a symfs is given */ 1503 /* skip the locally configured cache if a symfs is given */
1499 if (symbol_conf.symfs[0] || 1504 if (symbol_conf.symfs[0] ||
1500 (dso__build_id_filename(self, name, size) == NULL)) { 1505 (dso__build_id_filename(dso, name, size) == NULL)) {
1501 continue; 1506 continue;
1502 } 1507 }
1503 break; 1508 break;
1504 case SYMTAB__FEDORA_DEBUGINFO: 1509 case SYMTAB__FEDORA_DEBUGINFO:
1505 snprintf(name, size, "%s/usr/lib/debug%s.debug", 1510 snprintf(name, size, "%s/usr/lib/debug%s.debug",
1506 symbol_conf.symfs, self->long_name); 1511 symbol_conf.symfs, dso->long_name);
1507 break; 1512 break;
1508 case SYMTAB__UBUNTU_DEBUGINFO: 1513 case SYMTAB__UBUNTU_DEBUGINFO:
1509 snprintf(name, size, "%s/usr/lib/debug%s", 1514 snprintf(name, size, "%s/usr/lib/debug%s",
1510 symbol_conf.symfs, self->long_name); 1515 symbol_conf.symfs, dso->long_name);
1511 break; 1516 break;
1512 case SYMTAB__BUILDID_DEBUGINFO: { 1517 case SYMTAB__BUILDID_DEBUGINFO: {
1513 char build_id_hex[BUILD_ID_SIZE * 2 + 1]; 1518 char build_id_hex[BUILD_ID_SIZE * 2 + 1];
1514 1519
1515 if (!self->has_build_id) 1520 if (!dso->has_build_id)
1516 continue; 1521 continue;
1517 1522
1518 build_id__sprintf(self->build_id, 1523 build_id__sprintf(dso->build_id,
1519 sizeof(self->build_id), 1524 sizeof(dso->build_id),
1520 build_id_hex); 1525 build_id_hex);
1521 snprintf(name, size, 1526 snprintf(name, size,
1522 "%s/usr/lib/debug/.build-id/%.2s/%s.debug", 1527 "%s/usr/lib/debug/.build-id/%.2s/%s.debug",
@@ -1525,7 +1530,7 @@ restart:
1525 break; 1530 break;
1526 case SYMTAB__SYSTEM_PATH_DSO: 1531 case SYMTAB__SYSTEM_PATH_DSO:
1527 snprintf(name, size, "%s%s", 1532 snprintf(name, size, "%s%s",
1528 symbol_conf.symfs, self->long_name); 1533 symbol_conf.symfs, dso->long_name);
1529 break; 1534 break;
1530 case SYMTAB__GUEST_KMODULE: 1535 case SYMTAB__GUEST_KMODULE:
1531 if (map->groups && machine) 1536 if (map->groups && machine)
@@ -1533,12 +1538,12 @@ restart:
1533 else 1538 else
1534 root_dir = ""; 1539 root_dir = "";
1535 snprintf(name, size, "%s%s%s", symbol_conf.symfs, 1540 snprintf(name, size, "%s%s%s", symbol_conf.symfs,
1536 root_dir, self->long_name); 1541 root_dir, dso->long_name);
1537 break; 1542 break;
1538 1543
1539 case SYMTAB__SYSTEM_PATH_KMODULE: 1544 case SYMTAB__SYSTEM_PATH_KMODULE:
1540 snprintf(name, size, "%s%s", symbol_conf.symfs, 1545 snprintf(name, size, "%s%s", symbol_conf.symfs,
1541 self->long_name); 1546 dso->long_name);
1542 break; 1547 break;
1543 default:; 1548 default:;
1544 } 1549 }
@@ -1548,7 +1553,7 @@ restart:
1548 if (fd < 0) 1553 if (fd < 0)
1549 continue; 1554 continue;
1550 1555
1551 ret = dso__load_sym(self, map, name, fd, filter, 0, 1556 ret = dso__load_sym(dso, map, name, fd, filter, 0,
1552 want_symtab); 1557 want_symtab);
1553 close(fd); 1558 close(fd);
1554 1559
@@ -1560,7 +1565,8 @@ restart:
1560 continue; 1565 continue;
1561 1566
1562 if (ret > 0) { 1567 if (ret > 0) {
1563 int nr_plt = dso__synthesize_plt_symbols(self, map, filter); 1568 int nr_plt = dso__synthesize_plt_symbols(dso, map,
1569 filter);
1564 if (nr_plt > 0) 1570 if (nr_plt > 0)
1565 ret += nr_plt; 1571 ret += nr_plt;
1566 break; 1572 break;
@@ -1577,17 +1583,17 @@ restart:
1577 } 1583 }
1578 1584
1579 free(name); 1585 free(name);
1580 if (ret < 0 && strstr(self->name, " (deleted)") != NULL) 1586 if (ret < 0 && strstr(dso->name, " (deleted)") != NULL)
1581 return 0; 1587 return 0;
1582 return ret; 1588 return ret;
1583} 1589}
1584 1590
1585struct map *map_groups__find_by_name(struct map_groups *self, 1591struct map *map_groups__find_by_name(struct map_groups *mg,
1586 enum map_type type, const char *name) 1592 enum map_type type, const char *name)
1587{ 1593{
1588 struct rb_node *nd; 1594 struct rb_node *nd;
1589 1595
1590 for (nd = rb_first(&self->maps[type]); nd; nd = rb_next(nd)) { 1596 for (nd = rb_first(&mg->maps[type]); nd; nd = rb_next(nd)) {
1591 struct map *map = rb_entry(nd, struct map, rb_node); 1597 struct map *map = rb_entry(nd, struct map, rb_node);
1592 1598
1593 if (map->dso && strcmp(map->dso->short_name, name) == 0) 1599 if (map->dso && strcmp(map->dso->short_name, name) == 0)
@@ -1597,28 +1603,28 @@ struct map *map_groups__find_by_name(struct map_groups *self,
1597 return NULL; 1603 return NULL;
1598} 1604}
1599 1605
1600static int dso__kernel_module_get_build_id(struct dso *self, 1606static int dso__kernel_module_get_build_id(struct dso *dso,
1601 const char *root_dir) 1607 const char *root_dir)
1602{ 1608{
1603 char filename[PATH_MAX]; 1609 char filename[PATH_MAX];
1604 /* 1610 /*
1605 * kernel module short names are of the form "[module]" and 1611 * kernel module short names are of the form "[module]" and
1606 * we need just "module" here. 1612 * we need just "module" here.
1607 */ 1613 */
1608 const char *name = self->short_name + 1; 1614 const char *name = dso->short_name + 1;
1609 1615
1610 snprintf(filename, sizeof(filename), 1616 snprintf(filename, sizeof(filename),
1611 "%s/sys/module/%.*s/notes/.note.gnu.build-id", 1617 "%s/sys/module/%.*s/notes/.note.gnu.build-id",
1612 root_dir, (int)strlen(name) - 1, name); 1618 root_dir, (int)strlen(name) - 1, name);
1613 1619
1614 if (sysfs__read_build_id(filename, self->build_id, 1620 if (sysfs__read_build_id(filename, dso->build_id,
1615 sizeof(self->build_id)) == 0) 1621 sizeof(dso->build_id)) == 0)
1616 self->has_build_id = true; 1622 dso->has_build_id = true;
1617 1623
1618 return 0; 1624 return 0;
1619} 1625}
1620 1626
1621static int map_groups__set_modules_path_dir(struct map_groups *self, 1627static int map_groups__set_modules_path_dir(struct map_groups *mg,
1622 const char *dir_name) 1628 const char *dir_name)
1623{ 1629{
1624 struct dirent *dent; 1630 struct dirent *dent;
@@ -1646,7 +1652,7 @@ static int map_groups__set_modules_path_dir(struct map_groups *self,
1646 1652
1647 snprintf(path, sizeof(path), "%s/%s", 1653 snprintf(path, sizeof(path), "%s/%s",
1648 dir_name, dent->d_name); 1654 dir_name, dent->d_name);
1649 ret = map_groups__set_modules_path_dir(self, path); 1655 ret = map_groups__set_modules_path_dir(mg, path);
1650 if (ret < 0) 1656 if (ret < 0)
1651 goto out; 1657 goto out;
1652 } else { 1658 } else {
@@ -1661,7 +1667,8 @@ static int map_groups__set_modules_path_dir(struct map_groups *self,
1661 (int)(dot - dent->d_name), dent->d_name); 1667 (int)(dot - dent->d_name), dent->d_name);
1662 1668
1663 strxfrchar(dso_name, '-', '_'); 1669 strxfrchar(dso_name, '-', '_');
1664 map = map_groups__find_by_name(self, MAP__FUNCTION, dso_name); 1670 map = map_groups__find_by_name(mg, MAP__FUNCTION,
1671 dso_name);
1665 if (map == NULL) 1672 if (map == NULL)
1666 continue; 1673 continue;
1667 1674
@@ -1711,20 +1718,20 @@ static char *get_kernel_version(const char *root_dir)
1711 return strdup(name); 1718 return strdup(name);
1712} 1719}
1713 1720
1714static int machine__set_modules_path(struct machine *self) 1721static int machine__set_modules_path(struct machine *machine)
1715{ 1722{
1716 char *version; 1723 char *version;
1717 char modules_path[PATH_MAX]; 1724 char modules_path[PATH_MAX];
1718 1725
1719 version = get_kernel_version(self->root_dir); 1726 version = get_kernel_version(machine->root_dir);
1720 if (!version) 1727 if (!version)
1721 return -1; 1728 return -1;
1722 1729
1723 snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s/kernel", 1730 snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s/kernel",
1724 self->root_dir, version); 1731 machine->root_dir, version);
1725 free(version); 1732 free(version);
1726 1733
1727 return map_groups__set_modules_path_dir(&self->kmaps, modules_path); 1734 return map_groups__set_modules_path_dir(&machine->kmaps, modules_path);
1728} 1735}
1729 1736
1730/* 1737/*
@@ -1734,23 +1741,23 @@ static int machine__set_modules_path(struct machine *self)
1734 */ 1741 */
1735static struct map *map__new2(u64 start, struct dso *dso, enum map_type type) 1742static struct map *map__new2(u64 start, struct dso *dso, enum map_type type)
1736{ 1743{
1737 struct map *self = calloc(1, (sizeof(*self) + 1744 struct map *map = calloc(1, (sizeof(*map) +
1738 (dso->kernel ? sizeof(struct kmap) : 0))); 1745 (dso->kernel ? sizeof(struct kmap) : 0)));
1739 if (self != NULL) { 1746 if (map != NULL) {
1740 /* 1747 /*
1741 * ->end will be filled after we load all the symbols 1748 * ->end will be filled after we load all the symbols
1742 */ 1749 */
1743 map__init(self, type, start, 0, 0, dso); 1750 map__init(map, type, start, 0, 0, dso);
1744 } 1751 }
1745 1752
1746 return self; 1753 return map;
1747} 1754}
1748 1755
1749struct map *machine__new_module(struct machine *self, u64 start, 1756struct map *machine__new_module(struct machine *machine, u64 start,
1750 const char *filename) 1757 const char *filename)
1751{ 1758{
1752 struct map *map; 1759 struct map *map;
1753 struct dso *dso = __dsos__findnew(&self->kernel_dsos, filename); 1760 struct dso *dso = __dsos__findnew(&machine->kernel_dsos, filename);
1754 1761
1755 if (dso == NULL) 1762 if (dso == NULL)
1756 return NULL; 1763 return NULL;
@@ -1759,15 +1766,15 @@ struct map *machine__new_module(struct machine *self, u64 start,
1759 if (map == NULL) 1766 if (map == NULL)
1760 return NULL; 1767 return NULL;
1761 1768
1762 if (machine__is_host(self)) 1769 if (machine__is_host(machine))
1763 dso->symtab_type = SYMTAB__SYSTEM_PATH_KMODULE; 1770 dso->symtab_type = SYMTAB__SYSTEM_PATH_KMODULE;
1764 else 1771 else
1765 dso->symtab_type = SYMTAB__GUEST_KMODULE; 1772 dso->symtab_type = SYMTAB__GUEST_KMODULE;
1766 map_groups__insert(&self->kmaps, map); 1773 map_groups__insert(&machine->kmaps, map);
1767 return map; 1774 return map;
1768} 1775}
1769 1776
1770static int machine__create_modules(struct machine *self) 1777static int machine__create_modules(struct machine *machine)
1771{ 1778{
1772 char *line = NULL; 1779 char *line = NULL;
1773 size_t n; 1780 size_t n;
@@ -1776,10 +1783,10 @@ static int machine__create_modules(struct machine *self)
1776 const char *modules; 1783 const char *modules;
1777 char path[PATH_MAX]; 1784 char path[PATH_MAX];
1778 1785
1779 if (machine__is_default_guest(self)) 1786 if (machine__is_default_guest(machine))
1780 modules = symbol_conf.default_guest_modules; 1787 modules = symbol_conf.default_guest_modules;
1781 else { 1788 else {
1782 sprintf(path, "%s/proc/modules", self->root_dir); 1789 sprintf(path, "%s/proc/modules", machine->root_dir);
1783 modules = path; 1790 modules = path;
1784 } 1791 }
1785 1792
@@ -1815,16 +1822,16 @@ static int machine__create_modules(struct machine *self)
1815 *sep = '\0'; 1822 *sep = '\0';
1816 1823
1817 snprintf(name, sizeof(name), "[%s]", line); 1824 snprintf(name, sizeof(name), "[%s]", line);
1818 map = machine__new_module(self, start, name); 1825 map = machine__new_module(machine, start, name);
1819 if (map == NULL) 1826 if (map == NULL)
1820 goto out_delete_line; 1827 goto out_delete_line;
1821 dso__kernel_module_get_build_id(map->dso, self->root_dir); 1828 dso__kernel_module_get_build_id(map->dso, machine->root_dir);
1822 } 1829 }
1823 1830
1824 free(line); 1831 free(line);
1825 fclose(file); 1832 fclose(file);
1826 1833
1827 return machine__set_modules_path(self); 1834 return machine__set_modules_path(machine);
1828 1835
1829out_delete_line: 1836out_delete_line:
1830 free(line); 1837 free(line);
@@ -1832,7 +1839,7 @@ out_failure:
1832 return -1; 1839 return -1;
1833} 1840}
1834 1841
1835int dso__load_vmlinux(struct dso *self, struct map *map, 1842int dso__load_vmlinux(struct dso *dso, struct map *map,
1836 const char *vmlinux, symbol_filter_t filter) 1843 const char *vmlinux, symbol_filter_t filter)
1837{ 1844{
1838 int err = -1, fd; 1845 int err = -1, fd;
@@ -1844,9 +1851,9 @@ int dso__load_vmlinux(struct dso *self, struct map *map,
1844 if (fd < 0) 1851 if (fd < 0)
1845 return -1; 1852 return -1;
1846 1853
1847 dso__set_long_name(self, (char *)vmlinux); 1854 dso__set_long_name(dso, (char *)vmlinux);
1848 dso__set_loaded(self, map->type); 1855 dso__set_loaded(dso, map->type);
1849 err = dso__load_sym(self, map, symfs_vmlinux, fd, filter, 0, 0); 1856 err = dso__load_sym(dso, map, symfs_vmlinux, fd, filter, 0, 0);
1850 close(fd); 1857 close(fd);
1851 1858
1852 if (err > 0) 1859 if (err > 0)
@@ -1855,7 +1862,7 @@ int dso__load_vmlinux(struct dso *self, struct map *map,
1855 return err; 1862 return err;
1856} 1863}
1857 1864
1858int dso__load_vmlinux_path(struct dso *self, struct map *map, 1865int dso__load_vmlinux_path(struct dso *dso, struct map *map,
1859 symbol_filter_t filter) 1866 symbol_filter_t filter)
1860{ 1867{
1861 int i, err = 0; 1868 int i, err = 0;
@@ -1864,20 +1871,20 @@ int dso__load_vmlinux_path(struct dso *self, struct map *map,
1864 pr_debug("Looking at the vmlinux_path (%d entries long)\n", 1871 pr_debug("Looking at the vmlinux_path (%d entries long)\n",
1865 vmlinux_path__nr_entries + 1); 1872 vmlinux_path__nr_entries + 1);
1866 1873
1867 filename = dso__build_id_filename(self, NULL, 0); 1874 filename = dso__build_id_filename(dso, NULL, 0);
1868 if (filename != NULL) { 1875 if (filename != NULL) {
1869 err = dso__load_vmlinux(self, map, filename, filter); 1876 err = dso__load_vmlinux(dso, map, filename, filter);
1870 if (err > 0) { 1877 if (err > 0) {
1871 dso__set_long_name(self, filename); 1878 dso__set_long_name(dso, filename);
1872 goto out; 1879 goto out;
1873 } 1880 }
1874 free(filename); 1881 free(filename);
1875 } 1882 }
1876 1883
1877 for (i = 0; i < vmlinux_path__nr_entries; ++i) { 1884 for (i = 0; i < vmlinux_path__nr_entries; ++i) {
1878 err = dso__load_vmlinux(self, map, vmlinux_path[i], filter); 1885 err = dso__load_vmlinux(dso, map, vmlinux_path[i], filter);
1879 if (err > 0) { 1886 if (err > 0) {
1880 dso__set_long_name(self, strdup(vmlinux_path[i])); 1887 dso__set_long_name(dso, strdup(vmlinux_path[i]));
1881 break; 1888 break;
1882 } 1889 }
1883 } 1890 }
@@ -1885,7 +1892,7 @@ out:
1885 return err; 1892 return err;
1886} 1893}
1887 1894
1888static int dso__load_kernel_sym(struct dso *self, struct map *map, 1895static int dso__load_kernel_sym(struct dso *dso, struct map *map,
1889 symbol_filter_t filter) 1896 symbol_filter_t filter)
1890{ 1897{
1891 int err; 1898 int err;
@@ -1912,10 +1919,10 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map,
1912 } 1919 }
1913 1920
1914 if (symbol_conf.vmlinux_name != NULL) { 1921 if (symbol_conf.vmlinux_name != NULL) {
1915 err = dso__load_vmlinux(self, map, 1922 err = dso__load_vmlinux(dso, map,
1916 symbol_conf.vmlinux_name, filter); 1923 symbol_conf.vmlinux_name, filter);
1917 if (err > 0) { 1924 if (err > 0) {
1918 dso__set_long_name(self, 1925 dso__set_long_name(dso,
1919 strdup(symbol_conf.vmlinux_name)); 1926 strdup(symbol_conf.vmlinux_name));
1920 goto out_fixup; 1927 goto out_fixup;
1921 } 1928 }
@@ -1923,7 +1930,7 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map,
1923 } 1930 }
1924 1931
1925 if (vmlinux_path != NULL) { 1932 if (vmlinux_path != NULL) {
1926 err = dso__load_vmlinux_path(self, map, filter); 1933 err = dso__load_vmlinux_path(dso, map, filter);
1927 if (err > 0) 1934 if (err > 0)
1928 goto out_fixup; 1935 goto out_fixup;
1929 } 1936 }
@@ -1937,13 +1944,13 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map,
1937 * we have a build-id, so check if it is the same as the running kernel, 1944 * we have a build-id, so check if it is the same as the running kernel,
1938 * using it if it is. 1945 * using it if it is.
1939 */ 1946 */
1940 if (self->has_build_id) { 1947 if (dso->has_build_id) {
1941 u8 kallsyms_build_id[BUILD_ID_SIZE]; 1948 u8 kallsyms_build_id[BUILD_ID_SIZE];
1942 char sbuild_id[BUILD_ID_SIZE * 2 + 1]; 1949 char sbuild_id[BUILD_ID_SIZE * 2 + 1];
1943 1950
1944 if (sysfs__read_build_id("/sys/kernel/notes", kallsyms_build_id, 1951 if (sysfs__read_build_id("/sys/kernel/notes", kallsyms_build_id,
1945 sizeof(kallsyms_build_id)) == 0) { 1952 sizeof(kallsyms_build_id)) == 0) {
1946 if (dso__build_id_equal(self, kallsyms_build_id)) { 1953 if (dso__build_id_equal(dso, kallsyms_build_id)) {
1947 kallsyms_filename = "/proc/kallsyms"; 1954 kallsyms_filename = "/proc/kallsyms";
1948 goto do_kallsyms; 1955 goto do_kallsyms;
1949 } 1956 }
@@ -1952,7 +1959,7 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map,
1952 * Now look if we have it on the build-id cache in 1959 * Now look if we have it on the build-id cache in
1953 * $HOME/.debug/[kernel.kallsyms]. 1960 * $HOME/.debug/[kernel.kallsyms].
1954 */ 1961 */
1955 build_id__sprintf(self->build_id, sizeof(self->build_id), 1962 build_id__sprintf(dso->build_id, sizeof(dso->build_id),
1956 sbuild_id); 1963 sbuild_id);
1957 1964
1958 if (asprintf(&kallsyms_allocated_filename, 1965 if (asprintf(&kallsyms_allocated_filename,
@@ -1979,7 +1986,7 @@ static int dso__load_kernel_sym(struct dso *self, struct map *map,
1979 } 1986 }
1980 1987
1981do_kallsyms: 1988do_kallsyms:
1982 err = dso__load_kallsyms(self, kallsyms_filename, map, filter); 1989 err = dso__load_kallsyms(dso, kallsyms_filename, map, filter);
1983 if (err > 0) 1990 if (err > 0)
1984 pr_debug("Using %s for symbols\n", kallsyms_filename); 1991 pr_debug("Using %s for symbols\n", kallsyms_filename);
1985 free(kallsyms_allocated_filename); 1992 free(kallsyms_allocated_filename);
@@ -1987,7 +1994,7 @@ do_kallsyms:
1987 if (err > 0) { 1994 if (err > 0) {
1988out_fixup: 1995out_fixup:
1989 if (kallsyms_filename != NULL) 1996 if (kallsyms_filename != NULL)
1990 dso__set_long_name(self, strdup("[kernel.kallsyms]")); 1997 dso__set_long_name(dso, strdup("[kernel.kallsyms]"));
1991 map__fixup_start(map); 1998 map__fixup_start(map);
1992 map__fixup_end(map); 1999 map__fixup_end(map);
1993 } 2000 }
@@ -1995,8 +2002,8 @@ out_fixup:
1995 return err; 2002 return err;
1996} 2003}
1997 2004
1998static int dso__load_guest_kernel_sym(struct dso *self, struct map *map, 2005static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map,
1999 symbol_filter_t filter) 2006 symbol_filter_t filter)
2000{ 2007{
2001 int err; 2008 int err;
2002 const char *kallsyms_filename = NULL; 2009 const char *kallsyms_filename = NULL;
@@ -2016,7 +2023,7 @@ static int dso__load_guest_kernel_sym(struct dso *self, struct map *map,
2016 * Or use file guest_kallsyms inputted by user on commandline 2023 * Or use file guest_kallsyms inputted by user on commandline
2017 */ 2024 */
2018 if (symbol_conf.default_guest_vmlinux_name != NULL) { 2025 if (symbol_conf.default_guest_vmlinux_name != NULL) {
2019 err = dso__load_vmlinux(self, map, 2026 err = dso__load_vmlinux(dso, map,
2020 symbol_conf.default_guest_vmlinux_name, filter); 2027 symbol_conf.default_guest_vmlinux_name, filter);
2021 goto out_try_fixup; 2028 goto out_try_fixup;
2022 } 2029 }
@@ -2029,7 +2036,7 @@ static int dso__load_guest_kernel_sym(struct dso *self, struct map *map,
2029 kallsyms_filename = path; 2036 kallsyms_filename = path;
2030 } 2037 }
2031 2038
2032 err = dso__load_kallsyms(self, kallsyms_filename, map, filter); 2039 err = dso__load_kallsyms(dso, kallsyms_filename, map, filter);
2033 if (err > 0) 2040 if (err > 0)
2034 pr_debug("Using %s for symbols\n", kallsyms_filename); 2041 pr_debug("Using %s for symbols\n", kallsyms_filename);
2035 2042
@@ -2037,7 +2044,7 @@ out_try_fixup:
2037 if (err > 0) { 2044 if (err > 0) {
2038 if (kallsyms_filename != NULL) { 2045 if (kallsyms_filename != NULL) {
2039 machine__mmap_name(machine, path, sizeof(path)); 2046 machine__mmap_name(machine, path, sizeof(path));
2040 dso__set_long_name(self, strdup(path)); 2047 dso__set_long_name(dso, strdup(path));
2041 } 2048 }
2042 map__fixup_start(map); 2049 map__fixup_start(map);
2043 map__fixup_end(map); 2050 map__fixup_end(map);
@@ -2090,12 +2097,12 @@ size_t __dsos__fprintf(struct list_head *head, FILE *fp)
2090 return ret; 2097 return ret;
2091} 2098}
2092 2099
2093size_t machines__fprintf_dsos(struct rb_root *self, FILE *fp) 2100size_t machines__fprintf_dsos(struct rb_root *machines, FILE *fp)
2094{ 2101{
2095 struct rb_node *nd; 2102 struct rb_node *nd;
2096 size_t ret = 0; 2103 size_t ret = 0;
2097 2104
2098 for (nd = rb_first(self); nd; nd = rb_next(nd)) { 2105 for (nd = rb_first(machines); nd; nd = rb_next(nd)) {
2099 struct machine *pos = rb_entry(nd, struct machine, rb_node); 2106 struct machine *pos = rb_entry(nd, struct machine, rb_node);
2100 ret += __dsos__fprintf(&pos->kernel_dsos, fp); 2107 ret += __dsos__fprintf(&pos->kernel_dsos, fp);
2101 ret += __dsos__fprintf(&pos->user_dsos, fp); 2108 ret += __dsos__fprintf(&pos->user_dsos, fp);
@@ -2119,18 +2126,20 @@ static size_t __dsos__fprintf_buildid(struct list_head *head, FILE *fp,
2119 return ret; 2126 return ret;
2120} 2127}
2121 2128
2122size_t machine__fprintf_dsos_buildid(struct machine *self, FILE *fp, bool with_hits) 2129size_t machine__fprintf_dsos_buildid(struct machine *machine, FILE *fp,
2130 bool with_hits)
2123{ 2131{
2124 return __dsos__fprintf_buildid(&self->kernel_dsos, fp, with_hits) + 2132 return __dsos__fprintf_buildid(&machine->kernel_dsos, fp, with_hits) +
2125 __dsos__fprintf_buildid(&self->user_dsos, fp, with_hits); 2133 __dsos__fprintf_buildid(&machine->user_dsos, fp, with_hits);
2126} 2134}
2127 2135
2128size_t machines__fprintf_dsos_buildid(struct rb_root *self, FILE *fp, bool with_hits) 2136size_t machines__fprintf_dsos_buildid(struct rb_root *machines,
2137 FILE *fp, bool with_hits)
2129{ 2138{
2130 struct rb_node *nd; 2139 struct rb_node *nd;
2131 size_t ret = 0; 2140 size_t ret = 0;
2132 2141
2133 for (nd = rb_first(self); nd; nd = rb_next(nd)) { 2142 for (nd = rb_first(machines); nd; nd = rb_next(nd)) {
2134 struct machine *pos = rb_entry(nd, struct machine, rb_node); 2143 struct machine *pos = rb_entry(nd, struct machine, rb_node);
2135 ret += machine__fprintf_dsos_buildid(pos, fp, with_hits); 2144 ret += machine__fprintf_dsos_buildid(pos, fp, with_hits);
2136 } 2145 }
@@ -2139,59 +2148,59 @@ size_t machines__fprintf_dsos_buildid(struct rb_root *self, FILE *fp, bool with_
2139 2148
2140struct dso *dso__new_kernel(const char *name) 2149struct dso *dso__new_kernel(const char *name)
2141{ 2150{
2142 struct dso *self = dso__new(name ?: "[kernel.kallsyms]"); 2151 struct dso *dso = dso__new(name ?: "[kernel.kallsyms]");
2143 2152
2144 if (self != NULL) { 2153 if (dso != NULL) {
2145 dso__set_short_name(self, "[kernel]"); 2154 dso__set_short_name(dso, "[kernel]");
2146 self->kernel = DSO_TYPE_KERNEL; 2155 dso->kernel = DSO_TYPE_KERNEL;
2147 } 2156 }
2148 2157
2149 return self; 2158 return dso;
2150} 2159}
2151 2160
2152static struct dso *dso__new_guest_kernel(struct machine *machine, 2161static struct dso *dso__new_guest_kernel(struct machine *machine,
2153 const char *name) 2162 const char *name)
2154{ 2163{
2155 char bf[PATH_MAX]; 2164 char bf[PATH_MAX];
2156 struct dso *self = dso__new(name ?: machine__mmap_name(machine, bf, sizeof(bf))); 2165 struct dso *dso = dso__new(name ?: machine__mmap_name(machine, bf,
2157 2166 sizeof(bf)));
2158 if (self != NULL) { 2167 if (dso != NULL) {
2159 dso__set_short_name(self, "[guest.kernel]"); 2168 dso__set_short_name(dso, "[guest.kernel]");
2160 self->kernel = DSO_TYPE_GUEST_KERNEL; 2169 dso->kernel = DSO_TYPE_GUEST_KERNEL;
2161 } 2170 }
2162 2171
2163 return self; 2172 return dso;
2164} 2173}
2165 2174
2166void dso__read_running_kernel_build_id(struct dso *self, struct machine *machine) 2175void dso__read_running_kernel_build_id(struct dso *dso, struct machine *machine)
2167{ 2176{
2168 char path[PATH_MAX]; 2177 char path[PATH_MAX];
2169 2178
2170 if (machine__is_default_guest(machine)) 2179 if (machine__is_default_guest(machine))
2171 return; 2180 return;
2172 sprintf(path, "%s/sys/kernel/notes", machine->root_dir); 2181 sprintf(path, "%s/sys/kernel/notes", machine->root_dir);
2173 if (sysfs__read_build_id(path, self->build_id, 2182 if (sysfs__read_build_id(path, dso->build_id,
2174 sizeof(self->build_id)) == 0) 2183 sizeof(dso->build_id)) == 0)
2175 self->has_build_id = true; 2184 dso->has_build_id = true;
2176} 2185}
2177 2186
2178static struct dso *machine__create_kernel(struct machine *self) 2187static struct dso *machine__create_kernel(struct machine *machine)
2179{ 2188{
2180 const char *vmlinux_name = NULL; 2189 const char *vmlinux_name = NULL;
2181 struct dso *kernel; 2190 struct dso *kernel;
2182 2191
2183 if (machine__is_host(self)) { 2192 if (machine__is_host(machine)) {
2184 vmlinux_name = symbol_conf.vmlinux_name; 2193 vmlinux_name = symbol_conf.vmlinux_name;
2185 kernel = dso__new_kernel(vmlinux_name); 2194 kernel = dso__new_kernel(vmlinux_name);
2186 } else { 2195 } else {
2187 if (machine__is_default_guest(self)) 2196 if (machine__is_default_guest(machine))
2188 vmlinux_name = symbol_conf.default_guest_vmlinux_name; 2197 vmlinux_name = symbol_conf.default_guest_vmlinux_name;
2189 kernel = dso__new_guest_kernel(self, vmlinux_name); 2198 kernel = dso__new_guest_kernel(machine, vmlinux_name);
2190 } 2199 }
2191 2200
2192 if (kernel != NULL) { 2201 if (kernel != NULL) {
2193 dso__read_running_kernel_build_id(kernel, self); 2202 dso__read_running_kernel_build_id(kernel, machine);
2194 dsos__add(&self->kernel_dsos, kernel); 2203 dsos__add(&machine->kernel_dsos, kernel);
2195 } 2204 }
2196 return kernel; 2205 return kernel;
2197} 2206}
@@ -2236,41 +2245,43 @@ static u64 machine__get_kernel_start_addr(struct machine *machine)
2236 return args.start; 2245 return args.start;
2237} 2246}
2238 2247
2239int __machine__create_kernel_maps(struct machine *self, struct dso *kernel) 2248int __machine__create_kernel_maps(struct machine *machine, struct dso *kernel)
2240{ 2249{
2241 enum map_type type; 2250 enum map_type type;
2242 u64 start = machine__get_kernel_start_addr(self); 2251 u64 start = machine__get_kernel_start_addr(machine);
2243 2252
2244 for (type = 0; type < MAP__NR_TYPES; ++type) { 2253 for (type = 0; type < MAP__NR_TYPES; ++type) {
2245 struct kmap *kmap; 2254 struct kmap *kmap;
2246 2255
2247 self->vmlinux_maps[type] = map__new2(start, kernel, type); 2256 machine->vmlinux_maps[type] = map__new2(start, kernel, type);
2248 if (self->vmlinux_maps[type] == NULL) 2257 if (machine->vmlinux_maps[type] == NULL)
2249 return -1; 2258 return -1;
2250 2259
2251 self->vmlinux_maps[type]->map_ip = 2260 machine->vmlinux_maps[type]->map_ip =
2252 self->vmlinux_maps[type]->unmap_ip = identity__map_ip; 2261 machine->vmlinux_maps[type]->unmap_ip =
2253 2262 identity__map_ip;
2254 kmap = map__kmap(self->vmlinux_maps[type]); 2263 kmap = map__kmap(machine->vmlinux_maps[type]);
2255 kmap->kmaps = &self->kmaps; 2264 kmap->kmaps = &machine->kmaps;
2256 map_groups__insert(&self->kmaps, self->vmlinux_maps[type]); 2265 map_groups__insert(&machine->kmaps,
2266 machine->vmlinux_maps[type]);
2257 } 2267 }
2258 2268
2259 return 0; 2269 return 0;
2260} 2270}
2261 2271
2262void machine__destroy_kernel_maps(struct machine *self) 2272void machine__destroy_kernel_maps(struct machine *machine)
2263{ 2273{
2264 enum map_type type; 2274 enum map_type type;
2265 2275
2266 for (type = 0; type < MAP__NR_TYPES; ++type) { 2276 for (type = 0; type < MAP__NR_TYPES; ++type) {
2267 struct kmap *kmap; 2277 struct kmap *kmap;
2268 2278
2269 if (self->vmlinux_maps[type] == NULL) 2279 if (machine->vmlinux_maps[type] == NULL)
2270 continue; 2280 continue;
2271 2281
2272 kmap = map__kmap(self->vmlinux_maps[type]); 2282 kmap = map__kmap(machine->vmlinux_maps[type]);
2273 map_groups__remove(&self->kmaps, self->vmlinux_maps[type]); 2283 map_groups__remove(&machine->kmaps,
2284 machine->vmlinux_maps[type]);
2274 if (kmap->ref_reloc_sym) { 2285 if (kmap->ref_reloc_sym) {
2275 /* 2286 /*
2276 * ref_reloc_sym is shared among all maps, so free just 2287 * ref_reloc_sym is shared among all maps, so free just
@@ -2284,25 +2295,25 @@ void machine__destroy_kernel_maps(struct machine *self)
2284 kmap->ref_reloc_sym = NULL; 2295 kmap->ref_reloc_sym = NULL;
2285 } 2296 }
2286 2297
2287 map__delete(self->vmlinux_maps[type]); 2298 map__delete(machine->vmlinux_maps[type]);
2288 self->vmlinux_maps[type] = NULL; 2299 machine->vmlinux_maps[type] = NULL;
2289 } 2300 }
2290} 2301}
2291 2302
2292int machine__create_kernel_maps(struct machine *self) 2303int machine__create_kernel_maps(struct machine *machine)
2293{ 2304{
2294 struct dso *kernel = machine__create_kernel(self); 2305 struct dso *kernel = machine__create_kernel(machine);
2295 2306
2296 if (kernel == NULL || 2307 if (kernel == NULL ||
2297 __machine__create_kernel_maps(self, kernel) < 0) 2308 __machine__create_kernel_maps(machine, kernel) < 0)
2298 return -1; 2309 return -1;
2299 2310
2300 if (symbol_conf.use_modules && machine__create_modules(self) < 0) 2311 if (symbol_conf.use_modules && machine__create_modules(machine) < 0)
2301 pr_debug("Problems creating module maps, continuing anyway...\n"); 2312 pr_debug("Problems creating module maps, continuing anyway...\n");
2302 /* 2313 /*
2303 * Now that we have all the maps created, just set the ->end of them: 2314 * Now that we have all the maps created, just set the ->end of them:
2304 */ 2315 */
2305 map_groups__fixup_end(&self->kmaps); 2316 map_groups__fixup_end(&machine->kmaps);
2306 return 0; 2317 return 0;
2307} 2318}
2308 2319
@@ -2366,11 +2377,11 @@ out_fail:
2366 return -1; 2377 return -1;
2367} 2378}
2368 2379
2369size_t machine__fprintf_vmlinux_path(struct machine *self, FILE *fp) 2380size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp)
2370{ 2381{
2371 int i; 2382 int i;
2372 size_t printed = 0; 2383 size_t printed = 0;
2373 struct dso *kdso = self->vmlinux_maps[MAP__FUNCTION]->dso; 2384 struct dso *kdso = machine->vmlinux_maps[MAP__FUNCTION]->dso;
2374 2385
2375 if (kdso->has_build_id) { 2386 if (kdso->has_build_id) {
2376 char filename[PATH_MAX]; 2387 char filename[PATH_MAX];
@@ -2467,9 +2478,9 @@ void symbol__exit(void)
2467 symbol_conf.initialized = false; 2478 symbol_conf.initialized = false;
2468} 2479}
2469 2480
2470int machines__create_kernel_maps(struct rb_root *self, pid_t pid) 2481int machines__create_kernel_maps(struct rb_root *machines, pid_t pid)
2471{ 2482{
2472 struct machine *machine = machines__findnew(self, pid); 2483 struct machine *machine = machines__findnew(machines, pid);
2473 2484
2474 if (machine == NULL) 2485 if (machine == NULL)
2475 return -1; 2486 return -1;
@@ -2520,7 +2531,7 @@ char *strxfrchar(char *s, char from, char to)
2520 return s; 2531 return s;
2521} 2532}
2522 2533
2523int machines__create_guest_kernel_maps(struct rb_root *self) 2534int machines__create_guest_kernel_maps(struct rb_root *machines)
2524{ 2535{
2525 int ret = 0; 2536 int ret = 0;
2526 struct dirent **namelist = NULL; 2537 struct dirent **namelist = NULL;
@@ -2531,7 +2542,7 @@ int machines__create_guest_kernel_maps(struct rb_root *self)
2531 if (symbol_conf.default_guest_vmlinux_name || 2542 if (symbol_conf.default_guest_vmlinux_name ||
2532 symbol_conf.default_guest_modules || 2543 symbol_conf.default_guest_modules ||
2533 symbol_conf.default_guest_kallsyms) { 2544 symbol_conf.default_guest_kallsyms) {
2534 machines__create_kernel_maps(self, DEFAULT_GUEST_KERNEL_ID); 2545 machines__create_kernel_maps(machines, DEFAULT_GUEST_KERNEL_ID);
2535 } 2546 }
2536 2547
2537 if (symbol_conf.guestmount) { 2548 if (symbol_conf.guestmount) {
@@ -2552,7 +2563,7 @@ int machines__create_guest_kernel_maps(struct rb_root *self)
2552 pr_debug("Can't access file %s\n", path); 2563 pr_debug("Can't access file %s\n", path);
2553 goto failure; 2564 goto failure;
2554 } 2565 }
2555 machines__create_kernel_maps(self, pid); 2566 machines__create_kernel_maps(machines, pid);
2556 } 2567 }
2557failure: 2568failure:
2558 free(namelist); 2569 free(namelist);
@@ -2561,23 +2572,23 @@ failure:
2561 return ret; 2572 return ret;
2562} 2573}
2563 2574
2564void machines__destroy_guest_kernel_maps(struct rb_root *self) 2575void machines__destroy_guest_kernel_maps(struct rb_root *machines)
2565{ 2576{
2566 struct rb_node *next = rb_first(self); 2577 struct rb_node *next = rb_first(machines);
2567 2578
2568 while (next) { 2579 while (next) {
2569 struct machine *pos = rb_entry(next, struct machine, rb_node); 2580 struct machine *pos = rb_entry(next, struct machine, rb_node);
2570 2581
2571 next = rb_next(&pos->rb_node); 2582 next = rb_next(&pos->rb_node);
2572 rb_erase(&pos->rb_node, self); 2583 rb_erase(&pos->rb_node, machines);
2573 machine__delete(pos); 2584 machine__delete(pos);
2574 } 2585 }
2575} 2586}
2576 2587
2577int machine__load_kallsyms(struct machine *self, const char *filename, 2588int machine__load_kallsyms(struct machine *machine, const char *filename,
2578 enum map_type type, symbol_filter_t filter) 2589 enum map_type type, symbol_filter_t filter)
2579{ 2590{
2580 struct map *map = self->vmlinux_maps[type]; 2591 struct map *map = machine->vmlinux_maps[type];
2581 int ret = dso__load_kallsyms(map->dso, filename, map, filter); 2592 int ret = dso__load_kallsyms(map->dso, filename, map, filter);
2582 2593
2583 if (ret > 0) { 2594 if (ret > 0) {
@@ -2587,16 +2598,16 @@ int machine__load_kallsyms(struct machine *self, const char *filename,
2587 * kernel, with modules between them, fixup the end of all 2598 * kernel, with modules between them, fixup the end of all
2588 * sections. 2599 * sections.
2589 */ 2600 */
2590 __map_groups__fixup_end(&self->kmaps, type); 2601 __map_groups__fixup_end(&machine->kmaps, type);
2591 } 2602 }
2592 2603
2593 return ret; 2604 return ret;
2594} 2605}
2595 2606
2596int machine__load_vmlinux_path(struct machine *self, enum map_type type, 2607int machine__load_vmlinux_path(struct machine *machine, enum map_type type,
2597 symbol_filter_t filter) 2608 symbol_filter_t filter)
2598{ 2609{
2599 struct map *map = self->vmlinux_maps[type]; 2610 struct map *map = machine->vmlinux_maps[type];
2600 int ret = dso__load_vmlinux_path(map->dso, map, filter); 2611 int ret = dso__load_vmlinux_path(map->dso, map, filter);
2601 2612
2602 if (ret > 0) { 2613 if (ret > 0) {
diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h
index 713b0b40cc4a..242de0101a86 100644
--- a/tools/perf/util/symbol.h
+++ b/tools/perf/util/symbol.h
@@ -62,7 +62,7 @@ struct symbol {
62 char name[0]; 62 char name[0];
63}; 63};
64 64
65void symbol__delete(struct symbol *self); 65void symbol__delete(struct symbol *sym);
66 66
67struct strlist; 67struct strlist;
68 68
@@ -96,9 +96,9 @@ struct symbol_conf {
96 96
97extern struct symbol_conf symbol_conf; 97extern struct symbol_conf symbol_conf;
98 98
99static inline void *symbol__priv(struct symbol *self) 99static inline void *symbol__priv(struct symbol *sym)
100{ 100{
101 return ((void *)self) - symbol_conf.priv_size; 101 return ((void *)sym) - symbol_conf.priv_size;
102} 102}
103 103
104struct ref_reloc_sym { 104struct ref_reloc_sym {
@@ -155,43 +155,45 @@ struct dso {
155 155
156struct dso *dso__new(const char *name); 156struct dso *dso__new(const char *name);
157struct dso *dso__new_kernel(const char *name); 157struct dso *dso__new_kernel(const char *name);
158void dso__delete(struct dso *self); 158void dso__delete(struct dso *dso);
159 159
160int dso__name_len(const struct dso *self); 160int dso__name_len(const struct dso *dso);
161 161
162bool dso__loaded(const struct dso *self, enum map_type type); 162bool dso__loaded(const struct dso *dso, enum map_type type);
163bool dso__sorted_by_name(const struct dso *self, enum map_type type); 163bool dso__sorted_by_name(const struct dso *dso, enum map_type type);
164 164
165static inline void dso__set_loaded(struct dso *self, enum map_type type) 165static inline void dso__set_loaded(struct dso *dso, enum map_type type)
166{ 166{
167 self->loaded |= (1 << type); 167 dso->loaded |= (1 << type);
168} 168}
169 169
170void dso__sort_by_name(struct dso *self, enum map_type type); 170void dso__sort_by_name(struct dso *dso, enum map_type type);
171 171
172struct dso *__dsos__findnew(struct list_head *head, const char *name); 172struct dso *__dsos__findnew(struct list_head *head, const char *name);
173 173
174int dso__load(struct dso *self, struct map *map, symbol_filter_t filter); 174int dso__load(struct dso *dso, struct map *map, symbol_filter_t filter);
175int dso__load_vmlinux(struct dso *self, struct map *map, 175int dso__load_vmlinux(struct dso *dso, struct map *map,
176 const char *vmlinux, symbol_filter_t filter); 176 const char *vmlinux, symbol_filter_t filter);
177int dso__load_vmlinux_path(struct dso *self, struct map *map, 177int dso__load_vmlinux_path(struct dso *dso, struct map *map,
178 symbol_filter_t filter); 178 symbol_filter_t filter);
179int dso__load_kallsyms(struct dso *self, const char *filename, struct map *map, 179int dso__load_kallsyms(struct dso *dso, const char *filename, struct map *map,
180 symbol_filter_t filter); 180 symbol_filter_t filter);
181int machine__load_kallsyms(struct machine *self, const char *filename, 181int machine__load_kallsyms(struct machine *machine, const char *filename,
182 enum map_type type, symbol_filter_t filter); 182 enum map_type type, symbol_filter_t filter);
183int machine__load_vmlinux_path(struct machine *self, enum map_type type, 183int machine__load_vmlinux_path(struct machine *machine, enum map_type type,
184 symbol_filter_t filter); 184 symbol_filter_t filter);
185 185
186size_t __dsos__fprintf(struct list_head *head, FILE *fp); 186size_t __dsos__fprintf(struct list_head *head, FILE *fp);
187 187
188size_t machine__fprintf_dsos_buildid(struct machine *self, FILE *fp, bool with_hits); 188size_t machine__fprintf_dsos_buildid(struct machine *machine,
189size_t machines__fprintf_dsos(struct rb_root *self, FILE *fp); 189 FILE *fp, bool with_hits);
190size_t machines__fprintf_dsos_buildid(struct rb_root *self, FILE *fp, bool with_hits); 190size_t machines__fprintf_dsos(struct rb_root *machines, FILE *fp);
191 191size_t machines__fprintf_dsos_buildid(struct rb_root *machines,
192size_t dso__fprintf_buildid(struct dso *self, FILE *fp); 192 FILE *fp, bool with_hits);
193size_t dso__fprintf_symbols_by_name(struct dso *self, enum map_type type, FILE *fp); 193size_t dso__fprintf_buildid(struct dso *dso, FILE *fp);
194size_t dso__fprintf(struct dso *self, enum map_type type, FILE *fp); 194size_t dso__fprintf_symbols_by_name(struct dso *dso,
195 enum map_type type, FILE *fp);
196size_t dso__fprintf(struct dso *dso, enum map_type type, FILE *fp);
195 197
196enum symtab_type { 198enum symtab_type {
197 SYMTAB__KALLSYMS = 0, 199 SYMTAB__KALLSYMS = 0,
@@ -207,34 +209,36 @@ enum symtab_type {
207 SYMTAB__NOT_FOUND, 209 SYMTAB__NOT_FOUND,
208}; 210};
209 211
210char dso__symtab_origin(const struct dso *self); 212char dso__symtab_origin(const struct dso *dso);
211void dso__set_long_name(struct dso *self, char *name); 213void dso__set_long_name(struct dso *dso, char *name);
212void dso__set_build_id(struct dso *self, void *build_id); 214void dso__set_build_id(struct dso *dso, void *build_id);
213void dso__read_running_kernel_build_id(struct dso *self, struct machine *machine); 215void dso__read_running_kernel_build_id(struct dso *dso,
214struct symbol *dso__find_symbol(struct dso *self, enum map_type type, u64 addr); 216 struct machine *machine);
215struct symbol *dso__find_symbol_by_name(struct dso *self, enum map_type type, 217struct symbol *dso__find_symbol(struct dso *dso, enum map_type type,
218 u64 addr);
219struct symbol *dso__find_symbol_by_name(struct dso *dso, enum map_type type,
216 const char *name); 220 const char *name);
217 221
218int filename__read_build_id(const char *filename, void *bf, size_t size); 222int filename__read_build_id(const char *filename, void *bf, size_t size);
219int sysfs__read_build_id(const char *filename, void *bf, size_t size); 223int sysfs__read_build_id(const char *filename, void *bf, size_t size);
220bool __dsos__read_build_ids(struct list_head *head, bool with_hits); 224bool __dsos__read_build_ids(struct list_head *head, bool with_hits);
221int build_id__sprintf(const u8 *self, int len, char *bf); 225int build_id__sprintf(const u8 *build_id, int len, char *bf);
222int kallsyms__parse(const char *filename, void *arg, 226int kallsyms__parse(const char *filename, void *arg,
223 int (*process_symbol)(void *arg, const char *name, 227 int (*process_symbol)(void *arg, const char *name,
224 char type, u64 start, u64 end)); 228 char type, u64 start, u64 end));
225 229
226void machine__destroy_kernel_maps(struct machine *self); 230void machine__destroy_kernel_maps(struct machine *machine);
227int __machine__create_kernel_maps(struct machine *self, struct dso *kernel); 231int __machine__create_kernel_maps(struct machine *machine, struct dso *kernel);
228int machine__create_kernel_maps(struct machine *self); 232int machine__create_kernel_maps(struct machine *machine);
229 233
230int machines__create_kernel_maps(struct rb_root *self, pid_t pid); 234int machines__create_kernel_maps(struct rb_root *machines, pid_t pid);
231int machines__create_guest_kernel_maps(struct rb_root *self); 235int machines__create_guest_kernel_maps(struct rb_root *machines);
232void machines__destroy_guest_kernel_maps(struct rb_root *self); 236void machines__destroy_guest_kernel_maps(struct rb_root *machines);
233 237
234int symbol__init(void); 238int symbol__init(void);
235void symbol__exit(void); 239void symbol__exit(void);
236bool symbol_type__is_a(char symbol_type, enum map_type map_type); 240bool symbol_type__is_a(char symbol_type, enum map_type map_type);
237 241
238size_t machine__fprintf_vmlinux_path(struct machine *self, FILE *fp); 242size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp);
239 243
240#endif /* __PERF_SYMBOL */ 244#endif /* __PERF_SYMBOL */
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-29 03:00:51 -0500