aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorIngo Molnar <mingo@kernel.org>2013-08-14 11:58:56 -0400
committerIngo Molnar <mingo@kernel.org>2013-08-14 11:58:56 -0400
commit6f1d657668ac3041b65265d3653d7e9172a0d603 (patch)
tree6e837c683783708637cc4caf9de759166c7469b7 /include
parentd4e4ab86bcba5a72779c43dc1459f71fea3d89c8 (diff)
parentd13508f9440e46dccac6a2dd48d51a73b2207482 (diff)
Merge branch 'timers/nohz-v3' of git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks into timers/nohz
Pull nohz improvements from Frederic Weisbecker: " It mostly contains fixes and full dynticks off-case optimizations. I believe that distros want to enable this feature so it seems important to optimize the case where the "nohz_full=" parameter is empty. ie: I'm trying to remove any performance regression that comes with NO_HZ_FULL=y when the feature is not used. This patchset improves the current situation a lot (off-case appears to be around 11% faster with hackbench, although I guess it may vary depending on the configuration but it should be significantly faster in any case) now there is still some work to do: I can still observe a remaining loss of 1.6% throughput seen with hackbench compared to CONFIG_NO_HZ_FULL=n. " Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'include')
-rw-r--r--include/asm-generic/vtime.h0
-rw-r--r--include/linux/context_tracking.h128
-rw-r--r--include/linux/context_tracking_state.h39
-rw-r--r--include/linux/hardirq.h117
-rw-r--r--include/linux/preempt_mask.h122
-rw-r--r--include/linux/tick.h45
-rw-r--r--include/linux/vtime.h74
-rw-r--r--include/trace/events/context_tracking.h58
8 files changed, 393 insertions, 190 deletions
diff --git a/include/asm-generic/vtime.h b/include/asm-generic/vtime.h
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/include/asm-generic/vtime.h
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index fc09d7b0dacf..158158704c30 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -2,100 +2,110 @@
2#define _LINUX_CONTEXT_TRACKING_H 2#define _LINUX_CONTEXT_TRACKING_H
3 3
4#include <linux/sched.h> 4#include <linux/sched.h>
5#include <linux/percpu.h>
6#include <linux/vtime.h> 5#include <linux/vtime.h>
6#include <linux/context_tracking_state.h>
7#include <asm/ptrace.h> 7#include <asm/ptrace.h>
8 8
9struct context_tracking {
10 /*
11 * When active is false, probes are unset in order
12 * to minimize overhead: TIF flags are cleared
13 * and calls to user_enter/exit are ignored. This
14 * may be further optimized using static keys.
15 */
16 bool active;
17 enum ctx_state {
18 IN_KERNEL = 0,
19 IN_USER,
20 } state;
21};
22
23static inline void __guest_enter(void)
24{
25 /*
26 * This is running in ioctl context so we can avoid
27 * the call to vtime_account() with its unnecessary idle check.
28 */
29 vtime_account_system(current);
30 current->flags |= PF_VCPU;
31}
32
33static inline void __guest_exit(void)
34{
35 /*
36 * This is running in ioctl context so we can avoid
37 * the call to vtime_account() with its unnecessary idle check.
38 */
39 vtime_account_system(current);
40 current->flags &= ~PF_VCPU;
41}
42 9
43#ifdef CONFIG_CONTEXT_TRACKING 10#ifdef CONFIG_CONTEXT_TRACKING
44DECLARE_PER_CPU(struct context_tracking, context_tracking); 11extern void context_tracking_cpu_set(int cpu);
45 12
46static inline bool context_tracking_in_user(void) 13extern void context_tracking_user_enter(void);
14extern void context_tracking_user_exit(void);
15extern void __context_tracking_task_switch(struct task_struct *prev,
16 struct task_struct *next);
17
18static inline void user_enter(void)
47{ 19{
48 return __this_cpu_read(context_tracking.state) == IN_USER; 20 if (static_key_false(&context_tracking_enabled))
49} 21 context_tracking_user_enter();
50 22
51static inline bool context_tracking_active(void) 23}
24static inline void user_exit(void)
52{ 25{
53 return __this_cpu_read(context_tracking.active); 26 if (static_key_false(&context_tracking_enabled))
27 context_tracking_user_exit();
54} 28}
55 29
56extern void user_enter(void);
57extern void user_exit(void);
58
59extern void guest_enter(void);
60extern void guest_exit(void);
61
62static inline enum ctx_state exception_enter(void) 30static inline enum ctx_state exception_enter(void)
63{ 31{
64 enum ctx_state prev_ctx; 32 enum ctx_state prev_ctx;
65 33
34 if (!static_key_false(&context_tracking_enabled))
35 return 0;
36
66 prev_ctx = this_cpu_read(context_tracking.state); 37 prev_ctx = this_cpu_read(context_tracking.state);
67 user_exit(); 38 context_tracking_user_exit();
68 39
69 return prev_ctx; 40 return prev_ctx;
70} 41}
71 42
72static inline void exception_exit(enum ctx_state prev_ctx) 43static inline void exception_exit(enum ctx_state prev_ctx)
73{ 44{
74 if (prev_ctx == IN_USER) 45 if (static_key_false(&context_tracking_enabled)) {
75 user_enter(); 46 if (prev_ctx == IN_USER)
47 context_tracking_user_enter();
48 }
76} 49}
77 50
78extern void context_tracking_task_switch(struct task_struct *prev, 51static inline void context_tracking_task_switch(struct task_struct *prev,
79 struct task_struct *next); 52 struct task_struct *next)
53{
54 if (static_key_false(&context_tracking_enabled))
55 __context_tracking_task_switch(prev, next);
56}
80#else 57#else
81static inline bool context_tracking_in_user(void) { return false; }
82static inline void user_enter(void) { } 58static inline void user_enter(void) { }
83static inline void user_exit(void) { } 59static inline void user_exit(void) { }
60static inline enum ctx_state exception_enter(void) { return 0; }
61static inline void exception_exit(enum ctx_state prev_ctx) { }
62static inline void context_tracking_task_switch(struct task_struct *prev,
63 struct task_struct *next) { }
64#endif /* !CONFIG_CONTEXT_TRACKING */
65
66
67#ifdef CONFIG_CONTEXT_TRACKING_FORCE
68extern void context_tracking_init(void);
69#else
70static inline void context_tracking_init(void) { }
71#endif /* CONFIG_CONTEXT_TRACKING_FORCE */
72
84 73
74#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
85static inline void guest_enter(void) 75static inline void guest_enter(void)
86{ 76{
87 __guest_enter(); 77 if (vtime_accounting_enabled())
78 vtime_guest_enter(current);
79 else
80 current->flags |= PF_VCPU;
88} 81}
89 82
90static inline void guest_exit(void) 83static inline void guest_exit(void)
91{ 84{
92 __guest_exit(); 85 if (vtime_accounting_enabled())
86 vtime_guest_exit(current);
87 else
88 current->flags &= ~PF_VCPU;
93} 89}
94 90
95static inline enum ctx_state exception_enter(void) { return 0; } 91#else
96static inline void exception_exit(enum ctx_state prev_ctx) { } 92static inline void guest_enter(void)
97static inline void context_tracking_task_switch(struct task_struct *prev, 93{
98 struct task_struct *next) { } 94 /*
99#endif /* !CONFIG_CONTEXT_TRACKING */ 95 * This is running in ioctl context so its safe
96 * to assume that it's the stime pending cputime
97 * to flush.
98 */
99 vtime_account_system(current);
100 current->flags |= PF_VCPU;
101}
102
103static inline void guest_exit(void)
104{
105 /* Flush the guest cputime we spent on the guest */
106 vtime_account_system(current);
107 current->flags &= ~PF_VCPU;
108}
109#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
100 110
101#endif 111#endif
diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h
new file mode 100644
index 000000000000..0f1979d0674f
--- /dev/null
+++ b/include/linux/context_tracking_state.h
@@ -0,0 +1,39 @@
1#ifndef _LINUX_CONTEXT_TRACKING_STATE_H
2#define _LINUX_CONTEXT_TRACKING_STATE_H
3
4#include <linux/percpu.h>
5#include <linux/static_key.h>
6
7struct context_tracking {
8 /*
9 * When active is false, probes are unset in order
10 * to minimize overhead: TIF flags are cleared
11 * and calls to user_enter/exit are ignored. This
12 * may be further optimized using static keys.
13 */
14 bool active;
15 enum ctx_state {
16 IN_KERNEL = 0,
17 IN_USER,
18 } state;
19};
20
21#ifdef CONFIG_CONTEXT_TRACKING
22extern struct static_key context_tracking_enabled;
23DECLARE_PER_CPU(struct context_tracking, context_tracking);
24
25static inline bool context_tracking_in_user(void)
26{
27 return __this_cpu_read(context_tracking.state) == IN_USER;
28}
29
30static inline bool context_tracking_active(void)
31{
32 return __this_cpu_read(context_tracking.active);
33}
34#else
35static inline bool context_tracking_in_user(void) { return false; }
36static inline bool context_tracking_active(void) { return false; }
37#endif /* CONFIG_CONTEXT_TRACKING */
38
39#endif
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 05bcc0903766..ccfe17c5c8da 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -1,126 +1,11 @@
1#ifndef LINUX_HARDIRQ_H 1#ifndef LINUX_HARDIRQ_H
2#define LINUX_HARDIRQ_H 2#define LINUX_HARDIRQ_H
3 3
4#include <linux/preempt.h> 4#include <linux/preempt_mask.h>
5#include <linux/lockdep.h> 5#include <linux/lockdep.h>
6#include <linux/ftrace_irq.h> 6#include <linux/ftrace_irq.h>
7#include <linux/vtime.h> 7#include <linux/vtime.h>
8#include <asm/hardirq.h>
9 8
10/*
11 * We put the hardirq and softirq counter into the preemption
12 * counter. The bitmask has the following meaning:
13 *
14 * - bits 0-7 are the preemption count (max preemption depth: 256)
15 * - bits 8-15 are the softirq count (max # of softirqs: 256)
16 *
17 * The hardirq count can in theory reach the same as NR_IRQS.
18 * In reality, the number of nested IRQS is limited to the stack
19 * size as well. For archs with over 1000 IRQS it is not practical
20 * to expect that they will all nest. We give a max of 10 bits for
21 * hardirq nesting. An arch may choose to give less than 10 bits.
22 * m68k expects it to be 8.
23 *
24 * - bits 16-25 are the hardirq count (max # of nested hardirqs: 1024)
25 * - bit 26 is the NMI_MASK
26 * - bit 27 is the PREEMPT_ACTIVE flag
27 *
28 * PREEMPT_MASK: 0x000000ff
29 * SOFTIRQ_MASK: 0x0000ff00
30 * HARDIRQ_MASK: 0x03ff0000
31 * NMI_MASK: 0x04000000
32 */
33#define PREEMPT_BITS 8
34#define SOFTIRQ_BITS 8
35#define NMI_BITS 1
36
37#define MAX_HARDIRQ_BITS 10
38
39#ifndef HARDIRQ_BITS
40# define HARDIRQ_BITS MAX_HARDIRQ_BITS
41#endif
42
43#if HARDIRQ_BITS > MAX_HARDIRQ_BITS
44#error HARDIRQ_BITS too high!
45#endif
46
47#define PREEMPT_SHIFT 0
48#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
49#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
50#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
51
52#define __IRQ_MASK(x) ((1UL << (x))-1)
53
54#define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
55#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
56#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
57#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
58
59#define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT)
60#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT)
61#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
62#define NMI_OFFSET (1UL << NMI_SHIFT)
63
64#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)
65
66#ifndef PREEMPT_ACTIVE
67#define PREEMPT_ACTIVE_BITS 1
68#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)
69#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT)
70#endif
71
72#if PREEMPT_ACTIVE < (1 << (NMI_SHIFT + NMI_BITS))
73#error PREEMPT_ACTIVE is too low!
74#endif
75
76#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
77#define softirq_count() (preempt_count() & SOFTIRQ_MASK)
78#define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
79 | NMI_MASK))
80
81/*
82 * Are we doing bottom half or hardware interrupt processing?
83 * Are we in a softirq context? Interrupt context?
84 * in_softirq - Are we currently processing softirq or have bh disabled?
85 * in_serving_softirq - Are we currently processing softirq?
86 */
87#define in_irq() (hardirq_count())
88#define in_softirq() (softirq_count())
89#define in_interrupt() (irq_count())
90#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)
91
92/*
93 * Are we in NMI context?
94 */
95#define in_nmi() (preempt_count() & NMI_MASK)
96
97#if defined(CONFIG_PREEMPT_COUNT)
98# define PREEMPT_CHECK_OFFSET 1
99#else
100# define PREEMPT_CHECK_OFFSET 0
101#endif
102
103/*
104 * Are we running in atomic context? WARNING: this macro cannot
105 * always detect atomic context; in particular, it cannot know about
106 * held spinlocks in non-preemptible kernels. Thus it should not be
107 * used in the general case to determine whether sleeping is possible.
108 * Do not use in_atomic() in driver code.
109 */
110#define in_atomic() ((preempt_count() & ~PREEMPT_ACTIVE) != 0)
111
112/*
113 * Check whether we were atomic before we did preempt_disable():
114 * (used by the scheduler, *after* releasing the kernel lock)
115 */
116#define in_atomic_preempt_off() \
117 ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)
118
119#ifdef CONFIG_PREEMPT_COUNT
120# define preemptible() (preempt_count() == 0 && !irqs_disabled())
121#else
122# define preemptible() 0
123#endif
124 9
125#if defined(CONFIG_SMP) || defined(CONFIG_GENERIC_HARDIRQS) 10#if defined(CONFIG_SMP) || defined(CONFIG_GENERIC_HARDIRQS)
126extern void synchronize_irq(unsigned int irq); 11extern void synchronize_irq(unsigned int irq);
diff --git a/include/linux/preempt_mask.h b/include/linux/preempt_mask.h
new file mode 100644
index 000000000000..931bc616219f
--- /dev/null
+++ b/include/linux/preempt_mask.h
@@ -0,0 +1,122 @@
1#ifndef LINUX_PREEMPT_MASK_H
2#define LINUX_PREEMPT_MASK_H
3
4#include <linux/preempt.h>
5#include <asm/hardirq.h>
6
7/*
8 * We put the hardirq and softirq counter into the preemption
9 * counter. The bitmask has the following meaning:
10 *
11 * - bits 0-7 are the preemption count (max preemption depth: 256)
12 * - bits 8-15 are the softirq count (max # of softirqs: 256)
13 *
14 * The hardirq count can in theory reach the same as NR_IRQS.
15 * In reality, the number of nested IRQS is limited to the stack
16 * size as well. For archs with over 1000 IRQS it is not practical
17 * to expect that they will all nest. We give a max of 10 bits for
18 * hardirq nesting. An arch may choose to give less than 10 bits.
19 * m68k expects it to be 8.
20 *
21 * - bits 16-25 are the hardirq count (max # of nested hardirqs: 1024)
22 * - bit 26 is the NMI_MASK
23 * - bit 27 is the PREEMPT_ACTIVE flag
24 *
25 * PREEMPT_MASK: 0x000000ff
26 * SOFTIRQ_MASK: 0x0000ff00
27 * HARDIRQ_MASK: 0x03ff0000
28 * NMI_MASK: 0x04000000
29 */
30#define PREEMPT_BITS 8
31#define SOFTIRQ_BITS 8
32#define NMI_BITS 1
33
34#define MAX_HARDIRQ_BITS 10
35
36#ifndef HARDIRQ_BITS
37# define HARDIRQ_BITS MAX_HARDIRQ_BITS
38#endif
39
40#if HARDIRQ_BITS > MAX_HARDIRQ_BITS
41#error HARDIRQ_BITS too high!
42#endif
43
44#define PREEMPT_SHIFT 0
45#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
46#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
47#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
48
49#define __IRQ_MASK(x) ((1UL << (x))-1)
50
51#define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
52#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
53#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
54#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
55
56#define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT)
57#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT)
58#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
59#define NMI_OFFSET (1UL << NMI_SHIFT)
60
61#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)
62
63#ifndef PREEMPT_ACTIVE
64#define PREEMPT_ACTIVE_BITS 1
65#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)
66#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT)
67#endif
68
69#if PREEMPT_ACTIVE < (1 << (NMI_SHIFT + NMI_BITS))
70#error PREEMPT_ACTIVE is too low!
71#endif
72
73#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
74#define softirq_count() (preempt_count() & SOFTIRQ_MASK)
75#define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
76 | NMI_MASK))
77
78/*
79 * Are we doing bottom half or hardware interrupt processing?
80 * Are we in a softirq context? Interrupt context?
81 * in_softirq - Are we currently processing softirq or have bh disabled?
82 * in_serving_softirq - Are we currently processing softirq?
83 */
84#define in_irq() (hardirq_count())
85#define in_softirq() (softirq_count())
86#define in_interrupt() (irq_count())
87#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)
88
89/*
90 * Are we in NMI context?
91 */
92#define in_nmi() (preempt_count() & NMI_MASK)
93
94#if defined(CONFIG_PREEMPT_COUNT)
95# define PREEMPT_CHECK_OFFSET 1
96#else
97# define PREEMPT_CHECK_OFFSET 0
98#endif
99
100/*
101 * Are we running in atomic context? WARNING: this macro cannot
102 * always detect atomic context; in particular, it cannot know about
103 * held spinlocks in non-preemptible kernels. Thus it should not be
104 * used in the general case to determine whether sleeping is possible.
105 * Do not use in_atomic() in driver code.
106 */
107#define in_atomic() ((preempt_count() & ~PREEMPT_ACTIVE) != 0)
108
109/*
110 * Check whether we were atomic before we did preempt_disable():
111 * (used by the scheduler, *after* releasing the kernel lock)
112 */
113#define in_atomic_preempt_off() \
114 ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)
115
116#ifdef CONFIG_PREEMPT_COUNT
117# define preemptible() (preempt_count() == 0 && !irqs_disabled())
118#else
119# define preemptible() 0
120#endif
121
122#endif /* LINUX_PREEMPT_MASK_H */
diff --git a/include/linux/tick.h b/include/linux/tick.h
index 62bd8b72873c..5128d33bbb39 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -10,6 +10,8 @@
10#include <linux/irqflags.h> 10#include <linux/irqflags.h>
11#include <linux/percpu.h> 11#include <linux/percpu.h>
12#include <linux/hrtimer.h> 12#include <linux/hrtimer.h>
13#include <linux/context_tracking_state.h>
14#include <linux/cpumask.h>
13 15
14#ifdef CONFIG_GENERIC_CLOCKEVENTS 16#ifdef CONFIG_GENERIC_CLOCKEVENTS
15 17
@@ -158,20 +160,51 @@ static inline u64 get_cpu_iowait_time_us(int cpu, u64 *unused) { return -1; }
158# endif /* !CONFIG_NO_HZ_COMMON */ 160# endif /* !CONFIG_NO_HZ_COMMON */
159 161
160#ifdef CONFIG_NO_HZ_FULL 162#ifdef CONFIG_NO_HZ_FULL
163extern bool tick_nohz_full_running;
164extern cpumask_var_t tick_nohz_full_mask;
165
166static inline bool tick_nohz_full_enabled(void)
167{
168 if (!static_key_false(&context_tracking_enabled))
169 return false;
170
171 return tick_nohz_full_running;
172}
173
174static inline bool tick_nohz_full_cpu(int cpu)
175{
176 if (!tick_nohz_full_enabled())
177 return false;
178
179 return cpumask_test_cpu(cpu, tick_nohz_full_mask);
180}
181
161extern void tick_nohz_init(void); 182extern void tick_nohz_init(void);
162extern int tick_nohz_full_cpu(int cpu); 183extern void __tick_nohz_full_check(void);
163extern void tick_nohz_full_check(void);
164extern void tick_nohz_full_kick(void); 184extern void tick_nohz_full_kick(void);
165extern void tick_nohz_full_kick_all(void); 185extern void tick_nohz_full_kick_all(void);
166extern void tick_nohz_task_switch(struct task_struct *tsk); 186extern void __tick_nohz_task_switch(struct task_struct *tsk);
167#else 187#else
168static inline void tick_nohz_init(void) { } 188static inline void tick_nohz_init(void) { }
169static inline int tick_nohz_full_cpu(int cpu) { return 0; } 189static inline bool tick_nohz_full_enabled(void) { return false; }
170static inline void tick_nohz_full_check(void) { } 190static inline bool tick_nohz_full_cpu(int cpu) { return false; }
191static inline void __tick_nohz_full_check(void) { }
171static inline void tick_nohz_full_kick(void) { } 192static inline void tick_nohz_full_kick(void) { }
172static inline void tick_nohz_full_kick_all(void) { } 193static inline void tick_nohz_full_kick_all(void) { }
173static inline void tick_nohz_task_switch(struct task_struct *tsk) { } 194static inline void __tick_nohz_task_switch(struct task_struct *tsk) { }
174#endif 195#endif
175 196
197static inline void tick_nohz_full_check(void)
198{
199 if (tick_nohz_full_enabled())
200 __tick_nohz_full_check();
201}
202
203static inline void tick_nohz_task_switch(struct task_struct *tsk)
204{
205 if (tick_nohz_full_enabled())
206 __tick_nohz_task_switch(tsk);
207}
208
176 209
177#endif 210#endif
diff --git a/include/linux/vtime.h b/include/linux/vtime.h
index b1dd2db80076..f5b72b364bda 100644
--- a/include/linux/vtime.h
+++ b/include/linux/vtime.h
@@ -1,18 +1,68 @@
1#ifndef _LINUX_KERNEL_VTIME_H 1#ifndef _LINUX_KERNEL_VTIME_H
2#define _LINUX_KERNEL_VTIME_H 2#define _LINUX_KERNEL_VTIME_H
3 3
4#include <linux/context_tracking_state.h>
5#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
6#include <asm/vtime.h>
7#endif
8
9
4struct task_struct; 10struct task_struct;
5 11
12/*
13 * vtime_accounting_enabled() definitions/declarations
14 */
15#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
16static inline bool vtime_accounting_enabled(void) { return true; }
17#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
18
19#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
20static inline bool vtime_accounting_enabled(void)
21{
22 if (static_key_false(&context_tracking_enabled)) {
23 if (context_tracking_active())
24 return true;
25 }
26
27 return false;
28}
29#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
30
31#ifndef CONFIG_VIRT_CPU_ACCOUNTING
32static inline bool vtime_accounting_enabled(void) { return false; }
33#endif /* !CONFIG_VIRT_CPU_ACCOUNTING */
34
35
36/*
37 * Common vtime APIs
38 */
6#ifdef CONFIG_VIRT_CPU_ACCOUNTING 39#ifdef CONFIG_VIRT_CPU_ACCOUNTING
40
41#ifdef __ARCH_HAS_VTIME_TASK_SWITCH
7extern void vtime_task_switch(struct task_struct *prev); 42extern void vtime_task_switch(struct task_struct *prev);
43#else
44extern void vtime_common_task_switch(struct task_struct *prev);
45static inline void vtime_task_switch(struct task_struct *prev)
46{
47 if (vtime_accounting_enabled())
48 vtime_common_task_switch(prev);
49}
50#endif /* __ARCH_HAS_VTIME_TASK_SWITCH */
51
8extern void vtime_account_system(struct task_struct *tsk); 52extern void vtime_account_system(struct task_struct *tsk);
9extern void vtime_account_idle(struct task_struct *tsk); 53extern void vtime_account_idle(struct task_struct *tsk);
10extern void vtime_account_user(struct task_struct *tsk); 54extern void vtime_account_user(struct task_struct *tsk);
11extern void vtime_account_irq_enter(struct task_struct *tsk);
12 55
13#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 56#ifdef __ARCH_HAS_VTIME_ACCOUNT
14static inline bool vtime_accounting_enabled(void) { return true; } 57extern void vtime_account_irq_enter(struct task_struct *tsk);
15#endif 58#else
59extern void vtime_common_account_irq_enter(struct task_struct *tsk);
60static inline void vtime_account_irq_enter(struct task_struct *tsk)
61{
62 if (vtime_accounting_enabled())
63 vtime_common_account_irq_enter(tsk);
64}
65#endif /* __ARCH_HAS_VTIME_ACCOUNT */
16 66
17#else /* !CONFIG_VIRT_CPU_ACCOUNTING */ 67#else /* !CONFIG_VIRT_CPU_ACCOUNTING */
18 68
@@ -20,14 +70,20 @@ static inline void vtime_task_switch(struct task_struct *prev) { }
20static inline void vtime_account_system(struct task_struct *tsk) { } 70static inline void vtime_account_system(struct task_struct *tsk) { }
21static inline void vtime_account_user(struct task_struct *tsk) { } 71static inline void vtime_account_user(struct task_struct *tsk) { }
22static inline void vtime_account_irq_enter(struct task_struct *tsk) { } 72static inline void vtime_account_irq_enter(struct task_struct *tsk) { }
23static inline bool vtime_accounting_enabled(void) { return false; } 73#endif /* !CONFIG_VIRT_CPU_ACCOUNTING */
24#endif
25 74
26#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 75#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
27extern void arch_vtime_task_switch(struct task_struct *tsk); 76extern void arch_vtime_task_switch(struct task_struct *tsk);
28extern void vtime_account_irq_exit(struct task_struct *tsk); 77extern void vtime_gen_account_irq_exit(struct task_struct *tsk);
29extern bool vtime_accounting_enabled(void); 78
79static inline void vtime_account_irq_exit(struct task_struct *tsk)
80{
81 if (vtime_accounting_enabled())
82 vtime_gen_account_irq_exit(tsk);
83}
84
30extern void vtime_user_enter(struct task_struct *tsk); 85extern void vtime_user_enter(struct task_struct *tsk);
86
31static inline void vtime_user_exit(struct task_struct *tsk) 87static inline void vtime_user_exit(struct task_struct *tsk)
32{ 88{
33 vtime_account_user(tsk); 89 vtime_account_user(tsk);
@@ -35,7 +91,7 @@ static inline void vtime_user_exit(struct task_struct *tsk)
35extern void vtime_guest_enter(struct task_struct *tsk); 91extern void vtime_guest_enter(struct task_struct *tsk);
36extern void vtime_guest_exit(struct task_struct *tsk); 92extern void vtime_guest_exit(struct task_struct *tsk);
37extern void vtime_init_idle(struct task_struct *tsk, int cpu); 93extern void vtime_init_idle(struct task_struct *tsk, int cpu);
38#else 94#else /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */
39static inline void vtime_account_irq_exit(struct task_struct *tsk) 95static inline void vtime_account_irq_exit(struct task_struct *tsk)
40{ 96{
41 /* On hard|softirq exit we always account to hard|softirq cputime */ 97 /* On hard|softirq exit we always account to hard|softirq cputime */
diff --git a/include/trace/events/context_tracking.h b/include/trace/events/context_tracking.h
new file mode 100644
index 000000000000..ce8007cf29cf
--- /dev/null
+++ b/include/trace/events/context_tracking.h
@@ -0,0 +1,58 @@
1#undef TRACE_SYSTEM
2#define TRACE_SYSTEM context_tracking
3
4#if !defined(_TRACE_CONTEXT_TRACKING_H) || defined(TRACE_HEADER_MULTI_READ)
5#define _TRACE_CONTEXT_TRACKING_H
6
7#include <linux/tracepoint.h>
8
9DECLARE_EVENT_CLASS(context_tracking_user,
10
11 TP_PROTO(int dummy),
12
13 TP_ARGS(dummy),
14
15 TP_STRUCT__entry(
16 __field( int, dummy )
17 ),
18
19 TP_fast_assign(
20 __entry->dummy = dummy;
21 ),
22
23 TP_printk("%s", "")
24);
25
26/**
27 * user_enter - called when the kernel resumes to userspace
28 * @dummy: dummy arg to make trace event macro happy
29 *
30 * This event occurs when the kernel resumes to userspace after
31 * an exception or a syscall.
32 */
33DEFINE_EVENT(context_tracking_user, user_enter,
34
35 TP_PROTO(int dummy),
36
37 TP_ARGS(dummy)
38);
39
40/**
41 * user_exit - called when userspace enters the kernel
42 * @dummy: dummy arg to make trace event macro happy
43 *
44 * This event occurs when userspace enters the kernel through
45 * an exception or a syscall.
46 */
47DEFINE_EVENT(context_tracking_user, user_exit,
48
49 TP_PROTO(int dummy),
50
51 TP_ARGS(dummy)
52);
53
54
55#endif /* _TRACE_CONTEXT_TRACKING_H */
56
57/* This part must be outside protection */
58#include <trace/define_trace.h>
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-27 05:12:20 -0500