aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux
diff options
context:
space:
mode:
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/dynamic_debug.h2
-rw-r--r--include/linux/ftrace.h35
-rw-r--r--include/linux/init.h14
-rw-r--r--include/linux/jump_label.h89
-rw-r--r--include/linux/jump_label_ref.h44
-rw-r--r--include/linux/kernel.h1
-rw-r--r--include/linux/perf_event.h121
-rw-r--r--include/linux/tracepoint.h22
8 files changed, 157 insertions, 171 deletions
diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h
index 0c9653f11c18..e747ecd48e1c 100644
--- a/include/linux/dynamic_debug.h
+++ b/include/linux/dynamic_debug.h
@@ -1,8 +1,6 @@
1#ifndef _DYNAMIC_DEBUG_H 1#ifndef _DYNAMIC_DEBUG_H
2#define _DYNAMIC_DEBUG_H 2#define _DYNAMIC_DEBUG_H
3 3
4#include <linux/jump_label.h>
5
6/* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which 4/* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which
7 * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They 5 * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They
8 * use independent hash functions, to reduce the chance of false positives. 6 * use independent hash functions, to reduce the chance of false positives.
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index ca29e03c1fac..9d88e1cb5dbb 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -29,9 +29,22 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
29 29
30typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip); 30typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip);
31 31
32struct ftrace_hash;
33
34enum {
35 FTRACE_OPS_FL_ENABLED = 1 << 0,
36 FTRACE_OPS_FL_GLOBAL = 1 << 1,
37 FTRACE_OPS_FL_DYNAMIC = 1 << 2,
38};
39
32struct ftrace_ops { 40struct ftrace_ops {
33 ftrace_func_t func; 41 ftrace_func_t func;
34 struct ftrace_ops *next; 42 struct ftrace_ops *next;
43 unsigned long flags;
44#ifdef CONFIG_DYNAMIC_FTRACE
45 struct ftrace_hash *notrace_hash;
46 struct ftrace_hash *filter_hash;
47#endif
35}; 48};
36 49
37extern int function_trace_stop; 50extern int function_trace_stop;
@@ -146,14 +159,13 @@ extern void unregister_ftrace_function_probe_all(char *glob);
146extern int ftrace_text_reserved(void *start, void *end); 159extern int ftrace_text_reserved(void *start, void *end);
147 160
148enum { 161enum {
149 FTRACE_FL_FREE = (1 << 0), 162 FTRACE_FL_ENABLED = (1 << 30),
150 FTRACE_FL_FAILED = (1 << 1), 163 FTRACE_FL_FREE = (1 << 31),
151 FTRACE_FL_FILTER = (1 << 2),
152 FTRACE_FL_ENABLED = (1 << 3),
153 FTRACE_FL_NOTRACE = (1 << 4),
154 FTRACE_FL_CONVERTED = (1 << 5),
155}; 164};
156 165
166#define FTRACE_FL_MASK (0x3UL << 30)
167#define FTRACE_REF_MAX ((1 << 30) - 1)
168
157struct dyn_ftrace { 169struct dyn_ftrace {
158 union { 170 union {
159 unsigned long ip; /* address of mcount call-site */ 171 unsigned long ip; /* address of mcount call-site */
@@ -167,7 +179,12 @@ struct dyn_ftrace {
167}; 179};
168 180
169int ftrace_force_update(void); 181int ftrace_force_update(void);
170void ftrace_set_filter(unsigned char *buf, int len, int reset); 182void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
183 int len, int reset);
184void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
185 int len, int reset);
186void ftrace_set_global_filter(unsigned char *buf, int len, int reset);
187void ftrace_set_global_notrace(unsigned char *buf, int len, int reset);
171 188
172int register_ftrace_command(struct ftrace_func_command *cmd); 189int register_ftrace_command(struct ftrace_func_command *cmd);
173int unregister_ftrace_command(struct ftrace_func_command *cmd); 190int unregister_ftrace_command(struct ftrace_func_command *cmd);
diff --git a/include/linux/init.h b/include/linux/init.h
index 577671c55153..9146f39cdddf 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -79,29 +79,29 @@
79#define __exitused __used 79#define __exitused __used
80#endif 80#endif
81 81
82#define __exit __section(.exit.text) __exitused __cold 82#define __exit __section(.exit.text) __exitused __cold notrace
83 83
84/* Used for HOTPLUG */ 84/* Used for HOTPLUG */
85#define __devinit __section(.devinit.text) __cold 85#define __devinit __section(.devinit.text) __cold notrace
86#define __devinitdata __section(.devinit.data) 86#define __devinitdata __section(.devinit.data)
87#define __devinitconst __section(.devinit.rodata) 87#define __devinitconst __section(.devinit.rodata)
88#define __devexit __section(.devexit.text) __exitused __cold 88#define __devexit __section(.devexit.text) __exitused __cold notrace
89#define __devexitdata __section(.devexit.data) 89#define __devexitdata __section(.devexit.data)
90#define __devexitconst __section(.devexit.rodata) 90#define __devexitconst __section(.devexit.rodata)
91 91
92/* Used for HOTPLUG_CPU */ 92/* Used for HOTPLUG_CPU */
93#define __cpuinit __section(.cpuinit.text) __cold 93#define __cpuinit __section(.cpuinit.text) __cold notrace
94#define __cpuinitdata __section(.cpuinit.data) 94#define __cpuinitdata __section(.cpuinit.data)
95#define __cpuinitconst __section(.cpuinit.rodata) 95#define __cpuinitconst __section(.cpuinit.rodata)
96#define __cpuexit __section(.cpuexit.text) __exitused __cold 96#define __cpuexit __section(.cpuexit.text) __exitused __cold notrace
97#define __cpuexitdata __section(.cpuexit.data) 97#define __cpuexitdata __section(.cpuexit.data)
98#define __cpuexitconst __section(.cpuexit.rodata) 98#define __cpuexitconst __section(.cpuexit.rodata)
99 99
100/* Used for MEMORY_HOTPLUG */ 100/* Used for MEMORY_HOTPLUG */
101#define __meminit __section(.meminit.text) __cold 101#define __meminit __section(.meminit.text) __cold notrace
102#define __meminitdata __section(.meminit.data) 102#define __meminitdata __section(.meminit.data)
103#define __meminitconst __section(.meminit.rodata) 103#define __meminitconst __section(.meminit.rodata)
104#define __memexit __section(.memexit.text) __exitused __cold 104#define __memexit __section(.memexit.text) __exitused __cold notrace
105#define __memexitdata __section(.memexit.data) 105#define __memexitdata __section(.memexit.data)
106#define __memexitconst __section(.memexit.rodata) 106#define __memexitconst __section(.memexit.rodata)
107 107
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 7880f18e4b86..83e745f3ead7 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -1,20 +1,43 @@
1#ifndef _LINUX_JUMP_LABEL_H 1#ifndef _LINUX_JUMP_LABEL_H
2#define _LINUX_JUMP_LABEL_H 2#define _LINUX_JUMP_LABEL_H
3 3
4#include <linux/types.h>
5#include <linux/compiler.h>
6
4#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 7#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
8
9struct jump_label_key {
10 atomic_t enabled;
11 struct jump_entry *entries;
12#ifdef CONFIG_MODULES
13 struct jump_label_mod *next;
14#endif
15};
16
5# include <asm/jump_label.h> 17# include <asm/jump_label.h>
6# define HAVE_JUMP_LABEL 18# define HAVE_JUMP_LABEL
7#endif 19#endif
8 20
9enum jump_label_type { 21enum jump_label_type {
22 JUMP_LABEL_DISABLE = 0,
10 JUMP_LABEL_ENABLE, 23 JUMP_LABEL_ENABLE,
11 JUMP_LABEL_DISABLE
12}; 24};
13 25
14struct module; 26struct module;
15 27
16#ifdef HAVE_JUMP_LABEL 28#ifdef HAVE_JUMP_LABEL
17 29
30#ifdef CONFIG_MODULES
31#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
32#else
33#define JUMP_LABEL_INIT {{ 0 }, NULL}
34#endif
35
36static __always_inline bool static_branch(struct jump_label_key *key)
37{
38 return arch_static_branch(key);
39}
40
18extern struct jump_entry __start___jump_table[]; 41extern struct jump_entry __start___jump_table[];
19extern struct jump_entry __stop___jump_table[]; 42extern struct jump_entry __stop___jump_table[];
20 43
@@ -23,37 +46,37 @@ extern void jump_label_unlock(void);
23extern void arch_jump_label_transform(struct jump_entry *entry, 46extern void arch_jump_label_transform(struct jump_entry *entry,
24 enum jump_label_type type); 47 enum jump_label_type type);
25extern void arch_jump_label_text_poke_early(jump_label_t addr); 48extern void arch_jump_label_text_poke_early(jump_label_t addr);
26extern void jump_label_update(unsigned long key, enum jump_label_type type);
27extern void jump_label_apply_nops(struct module *mod);
28extern int jump_label_text_reserved(void *start, void *end); 49extern int jump_label_text_reserved(void *start, void *end);
50extern void jump_label_inc(struct jump_label_key *key);
51extern void jump_label_dec(struct jump_label_key *key);
52extern bool jump_label_enabled(struct jump_label_key *key);
53extern void jump_label_apply_nops(struct module *mod);
29 54
30#define jump_label_enable(key) \ 55#else
31 jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
32 56
33#define jump_label_disable(key) \ 57#include <asm/atomic.h>
34 jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
35 58
36#else 59#define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
37 60
38#define JUMP_LABEL(key, label) \ 61struct jump_label_key {
39do { \ 62 atomic_t enabled;
40 if (unlikely(*key)) \ 63};
41 goto label; \
42} while (0)
43 64
44#define jump_label_enable(cond_var) \ 65static __always_inline bool static_branch(struct jump_label_key *key)
45do { \ 66{
46 *(cond_var) = 1; \ 67 if (unlikely(atomic_read(&key->enabled)))
47} while (0) 68 return true;
69 return false;
70}
48 71
49#define jump_label_disable(cond_var) \ 72static inline void jump_label_inc(struct jump_label_key *key)
50do { \ 73{
51 *(cond_var) = 0; \ 74 atomic_inc(&key->enabled);
52} while (0) 75}
53 76
54static inline int jump_label_apply_nops(struct module *mod) 77static inline void jump_label_dec(struct jump_label_key *key)
55{ 78{
56 return 0; 79 atomic_dec(&key->enabled);
57} 80}
58 81
59static inline int jump_label_text_reserved(void *start, void *end) 82static inline int jump_label_text_reserved(void *start, void *end)
@@ -64,16 +87,16 @@ static inline int jump_label_text_reserved(void *start, void *end)
64static inline void jump_label_lock(void) {} 87static inline void jump_label_lock(void) {}
65static inline void jump_label_unlock(void) {} 88static inline void jump_label_unlock(void) {}
66 89
67#endif 90static inline bool jump_label_enabled(struct jump_label_key *key)
91{
92 return !!atomic_read(&key->enabled);
93}
68 94
69#define COND_STMT(key, stmt) \ 95static inline int jump_label_apply_nops(struct module *mod)
70do { \ 96{
71 __label__ jl_enabled; \ 97 return 0;
72 JUMP_LABEL(key, jl_enabled); \ 98}
73 if (0) { \ 99
74jl_enabled: \ 100#endif
75 stmt; \
76 } \
77} while (0)
78 101
79#endif 102#endif
diff --git a/include/linux/jump_label_ref.h b/include/linux/jump_label_ref.h
deleted file mode 100644
index e5d012ad92c6..000000000000
--- a/include/linux/jump_label_ref.h
+++ /dev/null
@@ -1,44 +0,0 @@
1#ifndef _LINUX_JUMP_LABEL_REF_H
2#define _LINUX_JUMP_LABEL_REF_H
3
4#include <linux/jump_label.h>
5#include <asm/atomic.h>
6
7#ifdef HAVE_JUMP_LABEL
8
9static inline void jump_label_inc(atomic_t *key)
10{
11 if (atomic_add_return(1, key) == 1)
12 jump_label_enable(key);
13}
14
15static inline void jump_label_dec(atomic_t *key)
16{
17 if (atomic_dec_and_test(key))
18 jump_label_disable(key);
19}
20
21#else /* !HAVE_JUMP_LABEL */
22
23static inline void jump_label_inc(atomic_t *key)
24{
25 atomic_inc(key);
26}
27
28static inline void jump_label_dec(atomic_t *key)
29{
30 atomic_dec(key);
31}
32
33#undef JUMP_LABEL
34#define JUMP_LABEL(key, label) \
35do { \
36 if (unlikely(__builtin_choose_expr( \
37 __builtin_types_compatible_p(typeof(key), atomic_t *), \
38 atomic_read((atomic_t *)(key)), *(key)))) \
39 goto label; \
40} while (0)
41
42#endif /* HAVE_JUMP_LABEL */
43
44#endif /* _LINUX_JUMP_LABEL_REF_H */
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 00cec4dc0ae2..f37ba716ef8b 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -283,6 +283,7 @@ extern char *get_options(const char *str, int nints, int *ints);
283extern unsigned long long memparse(const char *ptr, char **retptr); 283extern unsigned long long memparse(const char *ptr, char **retptr);
284 284
285extern int core_kernel_text(unsigned long addr); 285extern int core_kernel_text(unsigned long addr);
286extern int core_kernel_data(unsigned long addr);
286extern int __kernel_text_address(unsigned long addr); 287extern int __kernel_text_address(unsigned long addr);
287extern int kernel_text_address(unsigned long addr); 288extern int kernel_text_address(unsigned long addr);
288extern int func_ptr_is_kernel_text(void *ptr); 289extern int func_ptr_is_kernel_text(void *ptr);
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index ee9f1e782800..3412684ce5d5 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -2,8 +2,8 @@
2 * Performance events: 2 * Performance events:
3 * 3 *
4 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de> 4 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009, Red Hat, Inc., Ingo Molnar 5 * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2009, Red Hat, Inc., Peter Zijlstra 6 * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
7 * 7 *
8 * Data type definitions, declarations, prototypes. 8 * Data type definitions, declarations, prototypes.
9 * 9 *
@@ -52,6 +52,8 @@ enum perf_hw_id {
52 PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4, 52 PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4,
53 PERF_COUNT_HW_BRANCH_MISSES = 5, 53 PERF_COUNT_HW_BRANCH_MISSES = 5,
54 PERF_COUNT_HW_BUS_CYCLES = 6, 54 PERF_COUNT_HW_BUS_CYCLES = 6,
55 PERF_COUNT_HW_STALLED_CYCLES_FRONTEND = 7,
56 PERF_COUNT_HW_STALLED_CYCLES_BACKEND = 8,
55 57
56 PERF_COUNT_HW_MAX, /* non-ABI */ 58 PERF_COUNT_HW_MAX, /* non-ABI */
57}; 59};
@@ -468,9 +470,9 @@ enum perf_callchain_context {
468 PERF_CONTEXT_MAX = (__u64)-4095, 470 PERF_CONTEXT_MAX = (__u64)-4095,
469}; 471};
470 472
471#define PERF_FLAG_FD_NO_GROUP (1U << 0) 473#define PERF_FLAG_FD_NO_GROUP (1U << 0)
472#define PERF_FLAG_FD_OUTPUT (1U << 1) 474#define PERF_FLAG_FD_OUTPUT (1U << 1)
473#define PERF_FLAG_PID_CGROUP (1U << 2) /* pid=cgroup id, per-cpu mode only */ 475#define PERF_FLAG_PID_CGROUP (1U << 2) /* pid=cgroup id, per-cpu mode only */
474 476
475#ifdef __KERNEL__ 477#ifdef __KERNEL__
476/* 478/*
@@ -484,9 +486,9 @@ enum perf_callchain_context {
484#endif 486#endif
485 487
486struct perf_guest_info_callbacks { 488struct perf_guest_info_callbacks {
487 int (*is_in_guest) (void); 489 int (*is_in_guest)(void);
488 int (*is_user_mode) (void); 490 int (*is_user_mode)(void);
489 unsigned long (*get_guest_ip) (void); 491 unsigned long (*get_guest_ip)(void);
490}; 492};
491 493
492#ifdef CONFIG_HAVE_HW_BREAKPOINT 494#ifdef CONFIG_HAVE_HW_BREAKPOINT
@@ -505,7 +507,7 @@ struct perf_guest_info_callbacks {
505#include <linux/ftrace.h> 507#include <linux/ftrace.h>
506#include <linux/cpu.h> 508#include <linux/cpu.h>
507#include <linux/irq_work.h> 509#include <linux/irq_work.h>
508#include <linux/jump_label_ref.h> 510#include <linux/jump_label.h>
509#include <asm/atomic.h> 511#include <asm/atomic.h>
510#include <asm/local.h> 512#include <asm/local.h>
511 513
@@ -652,19 +654,19 @@ struct pmu {
652 * Start the transaction, after this ->add() doesn't need to 654 * Start the transaction, after this ->add() doesn't need to
653 * do schedulability tests. 655 * do schedulability tests.
654 */ 656 */
655 void (*start_txn) (struct pmu *pmu); /* optional */ 657 void (*start_txn) (struct pmu *pmu); /* optional */
656 /* 658 /*
657 * If ->start_txn() disabled the ->add() schedulability test 659 * If ->start_txn() disabled the ->add() schedulability test
658 * then ->commit_txn() is required to perform one. On success 660 * then ->commit_txn() is required to perform one. On success
659 * the transaction is closed. On error the transaction is kept 661 * the transaction is closed. On error the transaction is kept
660 * open until ->cancel_txn() is called. 662 * open until ->cancel_txn() is called.
661 */ 663 */
662 int (*commit_txn) (struct pmu *pmu); /* optional */ 664 int (*commit_txn) (struct pmu *pmu); /* optional */
663 /* 665 /*
664 * Will cancel the transaction, assumes ->del() is called 666 * Will cancel the transaction, assumes ->del() is called
665 * for each successful ->add() during the transaction. 667 * for each successful ->add() during the transaction.
666 */ 668 */
667 void (*cancel_txn) (struct pmu *pmu); /* optional */ 669 void (*cancel_txn) (struct pmu *pmu); /* optional */
668}; 670};
669 671
670/** 672/**
@@ -712,15 +714,15 @@ typedef void (*perf_overflow_handler_t)(struct perf_event *, int,
712 struct pt_regs *regs); 714 struct pt_regs *regs);
713 715
714enum perf_group_flag { 716enum perf_group_flag {
715 PERF_GROUP_SOFTWARE = 0x1, 717 PERF_GROUP_SOFTWARE = 0x1,
716}; 718};
717 719
718#define SWEVENT_HLIST_BITS 8 720#define SWEVENT_HLIST_BITS 8
719#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS) 721#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
720 722
721struct swevent_hlist { 723struct swevent_hlist {
722 struct hlist_head heads[SWEVENT_HLIST_SIZE]; 724 struct hlist_head heads[SWEVENT_HLIST_SIZE];
723 struct rcu_head rcu_head; 725 struct rcu_head rcu_head;
724}; 726};
725 727
726#define PERF_ATTACH_CONTEXT 0x01 728#define PERF_ATTACH_CONTEXT 0x01
@@ -733,13 +735,13 @@ struct swevent_hlist {
733 * This is a per-cpu dynamically allocated data structure. 735 * This is a per-cpu dynamically allocated data structure.
734 */ 736 */
735struct perf_cgroup_info { 737struct perf_cgroup_info {
736 u64 time; 738 u64 time;
737 u64 timestamp; 739 u64 timestamp;
738}; 740};
739 741
740struct perf_cgroup { 742struct perf_cgroup {
741 struct cgroup_subsys_state css; 743 struct cgroup_subsys_state css;
742 struct perf_cgroup_info *info; /* timing info, one per cpu */ 744 struct perf_cgroup_info *info; /* timing info, one per cpu */
743}; 745};
744#endif 746#endif
745 747
@@ -923,7 +925,7 @@ struct perf_event_context {
923 925
924/* 926/*
925 * Number of contexts where an event can trigger: 927 * Number of contexts where an event can trigger:
926 * task, softirq, hardirq, nmi. 928 * task, softirq, hardirq, nmi.
927 */ 929 */
928#define PERF_NR_CONTEXTS 4 930#define PERF_NR_CONTEXTS 4
929 931
@@ -1001,8 +1003,7 @@ struct perf_sample_data {
1001 struct perf_raw_record *raw; 1003 struct perf_raw_record *raw;
1002}; 1004};
1003 1005
1004static inline 1006static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
1005void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
1006{ 1007{
1007 data->addr = addr; 1008 data->addr = addr;
1008 data->raw = NULL; 1009 data->raw = NULL;
@@ -1034,13 +1035,12 @@ static inline int is_software_event(struct perf_event *event)
1034 return event->pmu->task_ctx_nr == perf_sw_context; 1035 return event->pmu->task_ctx_nr == perf_sw_context;
1035} 1036}
1036 1037
1037extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; 1038extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
1038 1039
1039extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64); 1040extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64);
1040 1041
1041#ifndef perf_arch_fetch_caller_regs 1042#ifndef perf_arch_fetch_caller_regs
1042static inline void 1043static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
1043perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
1044#endif 1044#endif
1045 1045
1046/* 1046/*
@@ -1063,26 +1063,24 @@ perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
1063{ 1063{
1064 struct pt_regs hot_regs; 1064 struct pt_regs hot_regs;
1065 1065
1066 JUMP_LABEL(&perf_swevent_enabled[event_id], have_event); 1066 if (static_branch(&perf_swevent_enabled[event_id])) {
1067 return; 1067 if (!regs) {
1068 1068 perf_fetch_caller_regs(&hot_regs);
1069have_event: 1069 regs = &hot_regs;
1070 if (!regs) { 1070 }
1071 perf_fetch_caller_regs(&hot_regs); 1071 __perf_sw_event(event_id, nr, nmi, regs, addr);
1072 regs = &hot_regs;
1073 } 1072 }
1074 __perf_sw_event(event_id, nr, nmi, regs, addr);
1075} 1073}
1076 1074
1077extern atomic_t perf_sched_events; 1075extern struct jump_label_key perf_sched_events;
1078 1076
1079static inline void perf_event_task_sched_in(struct task_struct *task) 1077static inline void perf_event_task_sched_in(struct task_struct *task)
1080{ 1078{
1081 COND_STMT(&perf_sched_events, __perf_event_task_sched_in(task)); 1079 if (static_branch(&perf_sched_events))
1080 __perf_event_task_sched_in(task);
1082} 1081}
1083 1082
1084static inline 1083static inline void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
1085void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
1086{ 1084{
1087 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0); 1085 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
1088 1086
@@ -1100,14 +1098,10 @@ extern void perf_event_fork(struct task_struct *tsk);
1100/* Callchains */ 1098/* Callchains */
1101DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); 1099DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
1102 1100
1103extern void perf_callchain_user(struct perf_callchain_entry *entry, 1101extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
1104 struct pt_regs *regs); 1102extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
1105extern void perf_callchain_kernel(struct perf_callchain_entry *entry,
1106 struct pt_regs *regs);
1107
1108 1103
1109static inline void 1104static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
1110perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
1111{ 1105{
1112 if (entry->nr < PERF_MAX_STACK_DEPTH) 1106 if (entry->nr < PERF_MAX_STACK_DEPTH)
1113 entry->ip[entry->nr++] = ip; 1107 entry->ip[entry->nr++] = ip;
@@ -1143,9 +1137,9 @@ extern void perf_tp_event(u64 addr, u64 count, void *record,
1143extern void perf_bp_event(struct perf_event *event, void *data); 1137extern void perf_bp_event(struct perf_event *event, void *data);
1144 1138
1145#ifndef perf_misc_flags 1139#ifndef perf_misc_flags
1146#define perf_misc_flags(regs) (user_mode(regs) ? PERF_RECORD_MISC_USER : \ 1140# define perf_misc_flags(regs) \
1147 PERF_RECORD_MISC_KERNEL) 1141 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
1148#define perf_instruction_pointer(regs) instruction_pointer(regs) 1142# define perf_instruction_pointer(regs) instruction_pointer(regs)
1149#endif 1143#endif
1150 1144
1151extern int perf_output_begin(struct perf_output_handle *handle, 1145extern int perf_output_begin(struct perf_output_handle *handle,
@@ -1180,9 +1174,9 @@ static inline void
1180perf_bp_event(struct perf_event *event, void *data) { } 1174perf_bp_event(struct perf_event *event, void *data) { }
1181 1175
1182static inline int perf_register_guest_info_callbacks 1176static inline int perf_register_guest_info_callbacks
1183(struct perf_guest_info_callbacks *callbacks) { return 0; } 1177(struct perf_guest_info_callbacks *callbacks) { return 0; }
1184static inline int perf_unregister_guest_info_callbacks 1178static inline int perf_unregister_guest_info_callbacks
1185(struct perf_guest_info_callbacks *callbacks) { return 0; } 1179(struct perf_guest_info_callbacks *callbacks) { return 0; }
1186 1180
1187static inline void perf_event_mmap(struct vm_area_struct *vma) { } 1181static inline void perf_event_mmap(struct vm_area_struct *vma) { }
1188static inline void perf_event_comm(struct task_struct *tsk) { } 1182static inline void perf_event_comm(struct task_struct *tsk) { }
@@ -1195,23 +1189,22 @@ static inline void perf_event_disable(struct perf_event *event) { }
1195static inline void perf_event_task_tick(void) { } 1189static inline void perf_event_task_tick(void) { }
1196#endif 1190#endif
1197 1191
1198#define perf_output_put(handle, x) \ 1192#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1199 perf_output_copy((handle), &(x), sizeof(x))
1200 1193
1201/* 1194/*
1202 * This has to have a higher priority than migration_notifier in sched.c. 1195 * This has to have a higher priority than migration_notifier in sched.c.
1203 */ 1196 */
1204#define perf_cpu_notifier(fn) \ 1197#define perf_cpu_notifier(fn) \
1205do { \ 1198do { \
1206 static struct notifier_block fn##_nb __cpuinitdata = \ 1199 static struct notifier_block fn##_nb __cpuinitdata = \
1207 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ 1200 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
1208 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ 1201 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \
1209 (void *)(unsigned long)smp_processor_id()); \ 1202 (void *)(unsigned long)smp_processor_id()); \
1210 fn(&fn##_nb, (unsigned long)CPU_STARTING, \ 1203 fn(&fn##_nb, (unsigned long)CPU_STARTING, \
1211 (void *)(unsigned long)smp_processor_id()); \ 1204 (void *)(unsigned long)smp_processor_id()); \
1212 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \ 1205 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \
1213 (void *)(unsigned long)smp_processor_id()); \ 1206 (void *)(unsigned long)smp_processor_id()); \
1214 register_cpu_notifier(&fn##_nb); \ 1207 register_cpu_notifier(&fn##_nb); \
1215} while (0) 1208} while (0)
1216 1209
1217#endif /* __KERNEL__ */ 1210#endif /* __KERNEL__ */
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 97c84a58efb8..d530a4460a0b 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -29,7 +29,7 @@ struct tracepoint_func {
29 29
30struct tracepoint { 30struct tracepoint {
31 const char *name; /* Tracepoint name */ 31 const char *name; /* Tracepoint name */
32 int state; /* State. */ 32 struct jump_label_key key;
33 void (*regfunc)(void); 33 void (*regfunc)(void);
34 void (*unregfunc)(void); 34 void (*unregfunc)(void);
35 struct tracepoint_func __rcu *funcs; 35 struct tracepoint_func __rcu *funcs;
@@ -146,9 +146,7 @@ void tracepoint_update_probe_range(struct tracepoint * const *begin,
146 extern struct tracepoint __tracepoint_##name; \ 146 extern struct tracepoint __tracepoint_##name; \
147 static inline void trace_##name(proto) \ 147 static inline void trace_##name(proto) \
148 { \ 148 { \
149 JUMP_LABEL(&__tracepoint_##name.state, do_trace); \ 149 if (static_branch(&__tracepoint_##name.key)) \
150 return; \
151do_trace: \
152 __DO_TRACE(&__tracepoint_##name, \ 150 __DO_TRACE(&__tracepoint_##name, \
153 TP_PROTO(data_proto), \ 151 TP_PROTO(data_proto), \
154 TP_ARGS(data_args), \ 152 TP_ARGS(data_args), \
@@ -176,14 +174,14 @@ do_trace: \
176 * structures, so we create an array of pointers that will be used for iteration 174 * structures, so we create an array of pointers that will be used for iteration
177 * on the tracepoints. 175 * on the tracepoints.
178 */ 176 */
179#define DEFINE_TRACE_FN(name, reg, unreg) \ 177#define DEFINE_TRACE_FN(name, reg, unreg) \
180 static const char __tpstrtab_##name[] \ 178 static const char __tpstrtab_##name[] \
181 __attribute__((section("__tracepoints_strings"))) = #name; \ 179 __attribute__((section("__tracepoints_strings"))) = #name; \
182 struct tracepoint __tracepoint_##name \ 180 struct tracepoint __tracepoint_##name \
183 __attribute__((section("__tracepoints"))) = \ 181 __attribute__((section("__tracepoints"))) = \
184 { __tpstrtab_##name, 0, reg, unreg, NULL }; \ 182 { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\
185 static struct tracepoint * const __tracepoint_ptr_##name __used \ 183 static struct tracepoint * const __tracepoint_ptr_##name __used \
186 __attribute__((section("__tracepoints_ptrs"))) = \ 184 __attribute__((section("__tracepoints_ptrs"))) = \
187 &__tracepoint_##name; 185 &__tracepoint_##name;
188 186
189#define DEFINE_TRACE(name) \ 187#define DEFINE_TRACE(name) \
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-28 00:52:47 -0400