aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86')
-rw-r--r--arch/x86/Kconfig3
-rw-r--r--arch/x86/include/asm/ftrace.h56
-rw-r--r--arch/x86/include/asm/kprobes.h1
-rw-r--r--arch/x86/include/asm/kvm.h16
-rw-r--r--arch/x86/include/asm/kvm_host.h16
-rw-r--r--arch/x86/include/asm/perf_event.h2
-rw-r--r--arch/x86/include/asm/perf_regs.h33
-rw-r--r--arch/x86/include/asm/processor.h2
-rw-r--r--arch/x86/include/asm/svm.h205
-rw-r--r--arch/x86/include/asm/uprobes.h3
-rw-r--r--arch/x86/include/asm/vmx.h127
-rw-r--r--arch/x86/kernel/Makefile2
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_uncore.c28
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_uncore.h6
-rw-r--r--arch/x86/kernel/entry_32.S74
-rw-r--r--arch/x86/kernel/entry_64.S130
-rw-r--r--arch/x86/kernel/ftrace.c73
-rw-r--r--arch/x86/kernel/kprobes.c67
-rw-r--r--arch/x86/kernel/perf_regs.c105
-rw-r--r--arch/x86/kernel/step.c53
-rw-r--r--arch/x86/kernel/uprobes.c52
-rw-r--r--arch/x86/kernel/x8664_ksyms_64.c6
-rw-r--r--arch/x86/kvm/trace.h89
23 files changed, 856 insertions, 293 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 20c49b8450b8..8ff1f56a0188 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -36,6 +36,7 @@ config X86
36 select HAVE_KRETPROBES 36 select HAVE_KRETPROBES
37 select HAVE_OPTPROBES 37 select HAVE_OPTPROBES
38 select HAVE_FTRACE_MCOUNT_RECORD 38 select HAVE_FTRACE_MCOUNT_RECORD
39 select HAVE_FENTRY if X86_64
39 select HAVE_C_RECORDMCOUNT 40 select HAVE_C_RECORDMCOUNT
40 select HAVE_DYNAMIC_FTRACE 41 select HAVE_DYNAMIC_FTRACE
41 select HAVE_FUNCTION_TRACER 42 select HAVE_FUNCTION_TRACER
@@ -60,6 +61,8 @@ config X86
60 select HAVE_MIXED_BREAKPOINTS_REGS 61 select HAVE_MIXED_BREAKPOINTS_REGS
61 select PERF_EVENTS 62 select PERF_EVENTS
62 select HAVE_PERF_EVENTS_NMI 63 select HAVE_PERF_EVENTS_NMI
64 select HAVE_PERF_REGS
65 select HAVE_PERF_USER_STACK_DUMP
63 select ANON_INODES 66 select ANON_INODES
64 select HAVE_ALIGNED_STRUCT_PAGE if SLUB && !M386 67 select HAVE_ALIGNED_STRUCT_PAGE if SLUB && !M386
65 select HAVE_CMPXCHG_LOCAL if !M386 68 select HAVE_CMPXCHG_LOCAL if !M386
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index b0767bc08740..9a25b522d377 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -3,38 +3,54 @@
3 3
4#ifdef __ASSEMBLY__ 4#ifdef __ASSEMBLY__
5 5
6 .macro MCOUNT_SAVE_FRAME 6 /* skip is set if the stack was already partially adjusted */
7 /* taken from glibc */ 7 .macro MCOUNT_SAVE_FRAME skip=0
8 subq $0x38, %rsp 8 /*
9 movq %rax, (%rsp) 9 * We add enough stack to save all regs.
10 movq %rcx, 8(%rsp) 10 */
11 movq %rdx, 16(%rsp) 11 subq $(SS+8-\skip), %rsp
12 movq %rsi, 24(%rsp) 12 movq %rax, RAX(%rsp)
13 movq %rdi, 32(%rsp) 13 movq %rcx, RCX(%rsp)
14 movq %r8, 40(%rsp) 14 movq %rdx, RDX(%rsp)
15 movq %r9, 48(%rsp) 15 movq %rsi, RSI(%rsp)
16 movq %rdi, RDI(%rsp)
17 movq %r8, R8(%rsp)
18 movq %r9, R9(%rsp)
19 /* Move RIP to its proper location */
20 movq SS+8(%rsp), %rdx
21 movq %rdx, RIP(%rsp)
16 .endm 22 .endm
17 23
18 .macro MCOUNT_RESTORE_FRAME 24 .macro MCOUNT_RESTORE_FRAME skip=0
19 movq 48(%rsp), %r9 25 movq R9(%rsp), %r9
20 movq 40(%rsp), %r8 26 movq R8(%rsp), %r8
21 movq 32(%rsp), %rdi 27 movq RDI(%rsp), %rdi
22 movq 24(%rsp), %rsi 28 movq RSI(%rsp), %rsi
23 movq 16(%rsp), %rdx 29 movq RDX(%rsp), %rdx
24 movq 8(%rsp), %rcx 30 movq RCX(%rsp), %rcx
25 movq (%rsp), %rax 31 movq RAX(%rsp), %rax
26 addq $0x38, %rsp 32 addq $(SS+8-\skip), %rsp
27 .endm 33 .endm
28 34
29#endif 35#endif
30 36
31#ifdef CONFIG_FUNCTION_TRACER 37#ifdef CONFIG_FUNCTION_TRACER
32#define MCOUNT_ADDR ((long)(mcount)) 38#ifdef CC_USING_FENTRY
39# define MCOUNT_ADDR ((long)(__fentry__))
40#else
41# define MCOUNT_ADDR ((long)(mcount))
42#endif
33#define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */ 43#define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */
34 44
45#ifdef CONFIG_DYNAMIC_FTRACE
46#define ARCH_SUPPORTS_FTRACE_OPS 1
47#define ARCH_SUPPORTS_FTRACE_SAVE_REGS
48#endif
49
35#ifndef __ASSEMBLY__ 50#ifndef __ASSEMBLY__
36extern void mcount(void); 51extern void mcount(void);
37extern atomic_t modifying_ftrace_code; 52extern atomic_t modifying_ftrace_code;
53extern void __fentry__(void);
38 54
39static inline unsigned long ftrace_call_adjust(unsigned long addr) 55static inline unsigned long ftrace_call_adjust(unsigned long addr)
40{ 56{
diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h
index 547882539157..d3ddd17405d0 100644
--- a/arch/x86/include/asm/kprobes.h
+++ b/arch/x86/include/asm/kprobes.h
@@ -27,6 +27,7 @@
27#include <asm/insn.h> 27#include <asm/insn.h>
28 28
29#define __ARCH_WANT_KPROBES_INSN_SLOT 29#define __ARCH_WANT_KPROBES_INSN_SLOT
30#define ARCH_SUPPORTS_KPROBES_ON_FTRACE
30 31
31struct pt_regs; 32struct pt_regs;
32struct kprobe; 33struct kprobe;
diff --git a/arch/x86/include/asm/kvm.h b/arch/x86/include/asm/kvm.h
index 246617efd67f..41e08cb6a092 100644
--- a/arch/x86/include/asm/kvm.h
+++ b/arch/x86/include/asm/kvm.h
@@ -9,6 +9,22 @@
9#include <linux/types.h> 9#include <linux/types.h>
10#include <linux/ioctl.h> 10#include <linux/ioctl.h>
11 11
12#define DE_VECTOR 0
13#define DB_VECTOR 1
14#define BP_VECTOR 3
15#define OF_VECTOR 4
16#define BR_VECTOR 5
17#define UD_VECTOR 6
18#define NM_VECTOR 7
19#define DF_VECTOR 8
20#define TS_VECTOR 10
21#define NP_VECTOR 11
22#define SS_VECTOR 12
23#define GP_VECTOR 13
24#define PF_VECTOR 14
25#define MF_VECTOR 16
26#define MC_VECTOR 18
27
12/* Select x86 specific features in <linux/kvm.h> */ 28/* Select x86 specific features in <linux/kvm.h> */
13#define __KVM_HAVE_PIT 29#define __KVM_HAVE_PIT
14#define __KVM_HAVE_IOAPIC 30#define __KVM_HAVE_IOAPIC
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 09155d64cf7e..1eaa6b056670 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -75,22 +75,6 @@
75#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) 75#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1))
76#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) 76#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
77 77
78#define DE_VECTOR 0
79#define DB_VECTOR 1
80#define BP_VECTOR 3
81#define OF_VECTOR 4
82#define BR_VECTOR 5
83#define UD_VECTOR 6
84#define NM_VECTOR 7
85#define DF_VECTOR 8
86#define TS_VECTOR 10
87#define NP_VECTOR 11
88#define SS_VECTOR 12
89#define GP_VECTOR 13
90#define PF_VECTOR 14
91#define MF_VECTOR 16
92#define MC_VECTOR 18
93
94#define SELECTOR_TI_MASK (1 << 2) 78#define SELECTOR_TI_MASK (1 << 2)
95#define SELECTOR_RPL_MASK 0x03 79#define SELECTOR_RPL_MASK 0x03
96 80
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index cb4e43bce98a..4fabcdf1cfa7 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -262,4 +262,6 @@ static inline void perf_check_microcode(void) { }
262 static inline void amd_pmu_disable_virt(void) { } 262 static inline void amd_pmu_disable_virt(void) { }
263#endif 263#endif
264 264
265#define arch_perf_out_copy_user copy_from_user_nmi
266
265#endif /* _ASM_X86_PERF_EVENT_H */ 267#endif /* _ASM_X86_PERF_EVENT_H */
diff --git a/arch/x86/include/asm/perf_regs.h b/arch/x86/include/asm/perf_regs.h
new file mode 100644
index 000000000000..3f2207bfd17b
--- /dev/null
+++ b/arch/x86/include/asm/perf_regs.h
@@ -0,0 +1,33 @@
1#ifndef _ASM_X86_PERF_REGS_H
2#define _ASM_X86_PERF_REGS_H
3
4enum perf_event_x86_regs {
5 PERF_REG_X86_AX,
6 PERF_REG_X86_BX,
7 PERF_REG_X86_CX,
8 PERF_REG_X86_DX,
9 PERF_REG_X86_SI,
10 PERF_REG_X86_DI,
11 PERF_REG_X86_BP,
12 PERF_REG_X86_SP,
13 PERF_REG_X86_IP,
14 PERF_REG_X86_FLAGS,
15 PERF_REG_X86_CS,
16 PERF_REG_X86_SS,
17 PERF_REG_X86_DS,
18 PERF_REG_X86_ES,
19 PERF_REG_X86_FS,
20 PERF_REG_X86_GS,
21 PERF_REG_X86_R8,
22 PERF_REG_X86_R9,
23 PERF_REG_X86_R10,
24 PERF_REG_X86_R11,
25 PERF_REG_X86_R12,
26 PERF_REG_X86_R13,
27 PERF_REG_X86_R14,
28 PERF_REG_X86_R15,
29
30 PERF_REG_X86_32_MAX = PERF_REG_X86_GS + 1,
31 PERF_REG_X86_64_MAX = PERF_REG_X86_R15 + 1,
32};
33#endif /* _ASM_X86_PERF_REGS_H */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index d048cad9bcad..433d2e5c98a7 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -759,6 +759,8 @@ static inline void update_debugctlmsr(unsigned long debugctlmsr)
759 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); 759 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
760} 760}
761 761
762extern void set_task_blockstep(struct task_struct *task, bool on);
763
762/* 764/*
763 * from system description table in BIOS. Mostly for MCA use, but 765 * from system description table in BIOS. Mostly for MCA use, but
764 * others may find it useful: 766 * others may find it useful:
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index f2b83bc7d784..cdf5674dd23a 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -1,6 +1,135 @@
1#ifndef __SVM_H 1#ifndef __SVM_H
2#define __SVM_H 2#define __SVM_H
3 3
4#define SVM_EXIT_READ_CR0 0x000
5#define SVM_EXIT_READ_CR3 0x003
6#define SVM_EXIT_READ_CR4 0x004
7#define SVM_EXIT_READ_CR8 0x008
8#define SVM_EXIT_WRITE_CR0 0x010
9#define SVM_EXIT_WRITE_CR3 0x013
10#define SVM_EXIT_WRITE_CR4 0x014
11#define SVM_EXIT_WRITE_CR8 0x018
12#define SVM_EXIT_READ_DR0 0x020
13#define SVM_EXIT_READ_DR1 0x021
14#define SVM_EXIT_READ_DR2 0x022
15#define SVM_EXIT_READ_DR3 0x023
16#define SVM_EXIT_READ_DR4 0x024
17#define SVM_EXIT_READ_DR5 0x025
18#define SVM_EXIT_READ_DR6 0x026
19#define SVM_EXIT_READ_DR7 0x027
20#define SVM_EXIT_WRITE_DR0 0x030
21#define SVM_EXIT_WRITE_DR1 0x031
22#define SVM_EXIT_WRITE_DR2 0x032
23#define SVM_EXIT_WRITE_DR3 0x033
24#define SVM_EXIT_WRITE_DR4 0x034
25#define SVM_EXIT_WRITE_DR5 0x035
26#define SVM_EXIT_WRITE_DR6 0x036
27#define SVM_EXIT_WRITE_DR7 0x037
28#define SVM_EXIT_EXCP_BASE 0x040
29#define SVM_EXIT_INTR 0x060
30#define SVM_EXIT_NMI 0x061
31#define SVM_EXIT_SMI 0x062
32#define SVM_EXIT_INIT 0x063
33#define SVM_EXIT_VINTR 0x064
34#define SVM_EXIT_CR0_SEL_WRITE 0x065
35#define SVM_EXIT_IDTR_READ 0x066
36#define SVM_EXIT_GDTR_READ 0x067
37#define SVM_EXIT_LDTR_READ 0x068
38#define SVM_EXIT_TR_READ 0x069
39#define SVM_EXIT_IDTR_WRITE 0x06a
40#define SVM_EXIT_GDTR_WRITE 0x06b
41#define SVM_EXIT_LDTR_WRITE 0x06c
42#define SVM_EXIT_TR_WRITE 0x06d
43#define SVM_EXIT_RDTSC 0x06e
44#define SVM_EXIT_RDPMC 0x06f
45#define SVM_EXIT_PUSHF 0x070
46#define SVM_EXIT_POPF 0x071
47#define SVM_EXIT_CPUID 0x072
48#define SVM_EXIT_RSM 0x073
49#define SVM_EXIT_IRET 0x074
50#define SVM_EXIT_SWINT 0x075
51#define SVM_EXIT_INVD 0x076
52#define SVM_EXIT_PAUSE 0x077
53#define SVM_EXIT_HLT 0x078
54#define SVM_EXIT_INVLPG 0x079
55#define SVM_EXIT_INVLPGA 0x07a
56#define SVM_EXIT_IOIO 0x07b
57#define SVM_EXIT_MSR 0x07c
58#define SVM_EXIT_TASK_SWITCH 0x07d
59#define SVM_EXIT_FERR_FREEZE 0x07e
60#define SVM_EXIT_SHUTDOWN 0x07f
61#define SVM_EXIT_VMRUN 0x080
62#define SVM_EXIT_VMMCALL 0x081
63#define SVM_EXIT_VMLOAD 0x082
64#define SVM_EXIT_VMSAVE 0x083
65#define SVM_EXIT_STGI 0x084
66#define SVM_EXIT_CLGI 0x085
67#define SVM_EXIT_SKINIT 0x086
68#define SVM_EXIT_RDTSCP 0x087
69#define SVM_EXIT_ICEBP 0x088
70#define SVM_EXIT_WBINVD 0x089
71#define SVM_EXIT_MONITOR 0x08a
72#define SVM_EXIT_MWAIT 0x08b
73#define SVM_EXIT_MWAIT_COND 0x08c
74#define SVM_EXIT_XSETBV 0x08d
75#define SVM_EXIT_NPF 0x400
76
77#define SVM_EXIT_ERR -1
78
79#define SVM_EXIT_REASONS \
80 { SVM_EXIT_READ_CR0, "read_cr0" }, \
81 { SVM_EXIT_READ_CR3, "read_cr3" }, \
82 { SVM_EXIT_READ_CR4, "read_cr4" }, \
83 { SVM_EXIT_READ_CR8, "read_cr8" }, \
84 { SVM_EXIT_WRITE_CR0, "write_cr0" }, \
85 { SVM_EXIT_WRITE_CR3, "write_cr3" }, \
86 { SVM_EXIT_WRITE_CR4, "write_cr4" }, \
87 { SVM_EXIT_WRITE_CR8, "write_cr8" }, \
88 { SVM_EXIT_READ_DR0, "read_dr0" }, \
89 { SVM_EXIT_READ_DR1, "read_dr1" }, \
90 { SVM_EXIT_READ_DR2, "read_dr2" }, \
91 { SVM_EXIT_READ_DR3, "read_dr3" }, \
92 { SVM_EXIT_WRITE_DR0, "write_dr0" }, \
93 { SVM_EXIT_WRITE_DR1, "write_dr1" }, \
94 { SVM_EXIT_WRITE_DR2, "write_dr2" }, \
95 { SVM_EXIT_WRITE_DR3, "write_dr3" }, \
96 { SVM_EXIT_WRITE_DR5, "write_dr5" }, \
97 { SVM_EXIT_WRITE_DR7, "write_dr7" }, \
98 { SVM_EXIT_EXCP_BASE + DB_VECTOR, "DB excp" }, \
99 { SVM_EXIT_EXCP_BASE + BP_VECTOR, "BP excp" }, \
100 { SVM_EXIT_EXCP_BASE + UD_VECTOR, "UD excp" }, \
101 { SVM_EXIT_EXCP_BASE + PF_VECTOR, "PF excp" }, \
102 { SVM_EXIT_EXCP_BASE + NM_VECTOR, "NM excp" }, \
103 { SVM_EXIT_EXCP_BASE + MC_VECTOR, "MC excp" }, \
104 { SVM_EXIT_INTR, "interrupt" }, \
105 { SVM_EXIT_NMI, "nmi" }, \
106 { SVM_EXIT_SMI, "smi" }, \
107 { SVM_EXIT_INIT, "init" }, \
108 { SVM_EXIT_VINTR, "vintr" }, \
109 { SVM_EXIT_CPUID, "cpuid" }, \
110 { SVM_EXIT_INVD, "invd" }, \
111 { SVM_EXIT_HLT, "hlt" }, \
112 { SVM_EXIT_INVLPG, "invlpg" }, \
113 { SVM_EXIT_INVLPGA, "invlpga" }, \
114 { SVM_EXIT_IOIO, "io" }, \
115 { SVM_EXIT_MSR, "msr" }, \
116 { SVM_EXIT_TASK_SWITCH, "task_switch" }, \
117 { SVM_EXIT_SHUTDOWN, "shutdown" }, \
118 { SVM_EXIT_VMRUN, "vmrun" }, \
119 { SVM_EXIT_VMMCALL, "hypercall" }, \
120 { SVM_EXIT_VMLOAD, "vmload" }, \
121 { SVM_EXIT_VMSAVE, "vmsave" }, \
122 { SVM_EXIT_STGI, "stgi" }, \
123 { SVM_EXIT_CLGI, "clgi" }, \
124 { SVM_EXIT_SKINIT, "skinit" }, \
125 { SVM_EXIT_WBINVD, "wbinvd" }, \
126 { SVM_EXIT_MONITOR, "monitor" }, \
127 { SVM_EXIT_MWAIT, "mwait" }, \
128 { SVM_EXIT_XSETBV, "xsetbv" }, \
129 { SVM_EXIT_NPF, "npf" }
130
131#ifdef __KERNEL__
132
4enum { 133enum {
5 INTERCEPT_INTR, 134 INTERCEPT_INTR,
6 INTERCEPT_NMI, 135 INTERCEPT_NMI,
@@ -264,81 +393,6 @@ struct __attribute__ ((__packed__)) vmcb {
264 393
265#define SVM_EXITINFO_REG_MASK 0x0F 394#define SVM_EXITINFO_REG_MASK 0x0F
266 395
267#define SVM_EXIT_READ_CR0 0x000
268#define SVM_EXIT_READ_CR3 0x003
269#define SVM_EXIT_READ_CR4 0x004
270#define SVM_EXIT_READ_CR8 0x008
271#define SVM_EXIT_WRITE_CR0 0x010
272#define SVM_EXIT_WRITE_CR3 0x013
273#define SVM_EXIT_WRITE_CR4 0x014
274#define SVM_EXIT_WRITE_CR8 0x018
275#define SVM_EXIT_READ_DR0 0x020
276#define SVM_EXIT_READ_DR1 0x021
277#define SVM_EXIT_READ_DR2 0x022
278#define SVM_EXIT_READ_DR3 0x023
279#define SVM_EXIT_READ_DR4 0x024
280#define SVM_EXIT_READ_DR5 0x025
281#define SVM_EXIT_READ_DR6 0x026
282#define SVM_EXIT_READ_DR7 0x027
283#define SVM_EXIT_WRITE_DR0 0x030
284#define SVM_EXIT_WRITE_DR1 0x031
285#define SVM_EXIT_WRITE_DR2 0x032
286#define SVM_EXIT_WRITE_DR3 0x033
287#define SVM_EXIT_WRITE_DR4 0x034
288#define SVM_EXIT_WRITE_DR5 0x035
289#define SVM_EXIT_WRITE_DR6 0x036
290#define SVM_EXIT_WRITE_DR7 0x037
291#define SVM_EXIT_EXCP_BASE 0x040
292#define SVM_EXIT_INTR 0x060
293#define SVM_EXIT_NMI 0x061
294#define SVM_EXIT_SMI 0x062
295#define SVM_EXIT_INIT 0x063
296#define SVM_EXIT_VINTR 0x064
297#define SVM_EXIT_CR0_SEL_WRITE 0x065
298#define SVM_EXIT_IDTR_READ 0x066
299#define SVM_EXIT_GDTR_READ 0x067
300#define SVM_EXIT_LDTR_READ 0x068
301#define SVM_EXIT_TR_READ 0x069
302#define SVM_EXIT_IDTR_WRITE 0x06a
303#define SVM_EXIT_GDTR_WRITE 0x06b
304#define SVM_EXIT_LDTR_WRITE 0x06c
305#define SVM_EXIT_TR_WRITE 0x06d
306#define SVM_EXIT_RDTSC 0x06e
307#define SVM_EXIT_RDPMC 0x06f
308#define SVM_EXIT_PUSHF 0x070
309#define SVM_EXIT_POPF 0x071
310#define SVM_EXIT_CPUID 0x072
311#define SVM_EXIT_RSM 0x073
312#define SVM_EXIT_IRET 0x074
313#define SVM_EXIT_SWINT 0x075
314#define SVM_EXIT_INVD 0x076
315#define SVM_EXIT_PAUSE 0x077
316#define SVM_EXIT_HLT 0x078
317#define SVM_EXIT_INVLPG 0x079
318#define SVM_EXIT_INVLPGA 0x07a
319#define SVM_EXIT_IOIO 0x07b
320#define SVM_EXIT_MSR 0x07c
321#define SVM_EXIT_TASK_SWITCH 0x07d
322#define SVM_EXIT_FERR_FREEZE 0x07e
323#define SVM_EXIT_SHUTDOWN 0x07f
324#define SVM_EXIT_VMRUN 0x080
325#define SVM_EXIT_VMMCALL 0x081
326#define SVM_EXIT_VMLOAD 0x082
327#define SVM_EXIT_VMSAVE 0x083
328#define SVM_EXIT_STGI 0x084
329#define SVM_EXIT_CLGI 0x085
330#define SVM_EXIT_SKINIT 0x086
331#define SVM_EXIT_RDTSCP 0x087
332#define SVM_EXIT_ICEBP 0x088
333#define SVM_EXIT_WBINVD 0x089
334#define SVM_EXIT_MONITOR 0x08a
335#define SVM_EXIT_MWAIT 0x08b
336#define SVM_EXIT_MWAIT_COND 0x08c
337#define SVM_EXIT_XSETBV 0x08d
338#define SVM_EXIT_NPF 0x400
339
340#define SVM_EXIT_ERR -1
341
342#define SVM_CR0_SELECTIVE_MASK (X86_CR0_TS | X86_CR0_MP) 396#define SVM_CR0_SELECTIVE_MASK (X86_CR0_TS | X86_CR0_MP)
343 397
344#define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda" 398#define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda"
@@ -350,3 +404,4 @@ struct __attribute__ ((__packed__)) vmcb {
350 404
351#endif 405#endif
352 406
407#endif
diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index f3971bbcd1de..8ff8be7835ab 100644
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -42,10 +42,11 @@ struct arch_uprobe {
42}; 42};
43 43
44struct arch_uprobe_task { 44struct arch_uprobe_task {
45 unsigned long saved_trap_nr;
46#ifdef CONFIG_X86_64 45#ifdef CONFIG_X86_64
47 unsigned long saved_scratch_register; 46 unsigned long saved_scratch_register;
48#endif 47#endif
48 unsigned int saved_trap_nr;
49 unsigned int saved_tf;
49}; 50};
50 51
51extern int arch_uprobe_analyze_insn(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long addr); 52extern int arch_uprobe_analyze_insn(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long addr);
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 74fcb963595b..36ec21c36d68 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -25,6 +25,88 @@
25 * 25 *
26 */ 26 */
27 27
28#define VMX_EXIT_REASONS_FAILED_VMENTRY 0x80000000
29
30#define EXIT_REASON_EXCEPTION_NMI 0
31#define EXIT_REASON_EXTERNAL_INTERRUPT 1
32#define EXIT_REASON_TRIPLE_FAULT 2
33
34#define EXIT_REASON_PENDING_INTERRUPT 7
35#define EXIT_REASON_NMI_WINDOW 8
36#define EXIT_REASON_TASK_SWITCH 9
37#define EXIT_REASON_CPUID 10
38#define EXIT_REASON_HLT 12
39#define EXIT_REASON_INVD 13
40#define EXIT_REASON_INVLPG 14
41#define EXIT_REASON_RDPMC 15
42#define EXIT_REASON_RDTSC 16
43#define EXIT_REASON_VMCALL 18
44#define EXIT_REASON_VMCLEAR 19
45#define EXIT_REASON_VMLAUNCH 20
46#define EXIT_REASON_VMPTRLD 21
47#define EXIT_REASON_VMPTRST 22
48#define EXIT_REASON_VMREAD 23
49#define EXIT_REASON_VMRESUME 24
50#define EXIT_REASON_VMWRITE 25
51#define EXIT_REASON_VMOFF 26
52#define EXIT_REASON_VMON 27
53#define EXIT_REASON_CR_ACCESS 28
54#define EXIT_REASON_DR_ACCESS 29
55#define EXIT_REASON_IO_INSTRUCTION 30
56#define EXIT_REASON_MSR_READ 31
57#define EXIT_REASON_MSR_WRITE 32
58#define EXIT_REASON_INVALID_STATE 33
59#define EXIT_REASON_MWAIT_INSTRUCTION 36
60#define EXIT_REASON_MONITOR_INSTRUCTION 39
61#define EXIT_REASON_PAUSE_INSTRUCTION 40
62#define EXIT_REASON_MCE_DURING_VMENTRY 41
63#define EXIT_REASON_TPR_BELOW_THRESHOLD 43
64#define EXIT_REASON_APIC_ACCESS 44
65#define EXIT_REASON_EPT_VIOLATION 48
66#define EXIT_REASON_EPT_MISCONFIG 49
67#define EXIT_REASON_WBINVD 54
68#define EXIT_REASON_XSETBV 55
69#define EXIT_REASON_INVPCID 58
70
71#define VMX_EXIT_REASONS \
72 { EXIT_REASON_EXCEPTION_NMI, "EXCEPTION_NMI" }, \
73 { EXIT_REASON_EXTERNAL_INTERRUPT, "EXTERNAL_INTERRUPT" }, \
74 { EXIT_REASON_TRIPLE_FAULT, "TRIPLE_FAULT" }, \
75 { EXIT_REASON_PENDING_INTERRUPT, "PENDING_INTERRUPT" }, \
76 { EXIT_REASON_NMI_WINDOW, "NMI_WINDOW" }, \
77 { EXIT_REASON_TASK_SWITCH, "TASK_SWITCH" }, \
78 { EXIT_REASON_CPUID, "CPUID" }, \
79 { EXIT_REASON_HLT, "HLT" }, \
80 { EXIT_REASON_INVLPG, "INVLPG" }, \
81 { EXIT_REASON_RDPMC, "RDPMC" }, \
82 { EXIT_REASON_RDTSC, "RDTSC" }, \
83 { EXIT_REASON_VMCALL, "VMCALL" }, \
84 { EXIT_REASON_VMCLEAR, "VMCLEAR" }, \
85 { EXIT_REASON_VMLAUNCH, "VMLAUNCH" }, \
86 { EXIT_REASON_VMPTRLD, "VMPTRLD" }, \
87 { EXIT_REASON_VMPTRST, "VMPTRST" }, \
88 { EXIT_REASON_VMREAD, "VMREAD" }, \
89 { EXIT_REASON_VMRESUME, "VMRESUME" }, \
90 { EXIT_REASON_VMWRITE, "VMWRITE" }, \
91 { EXIT_REASON_VMOFF, "VMOFF" }, \
92 { EXIT_REASON_VMON, "VMON" }, \
93 { EXIT_REASON_CR_ACCESS, "CR_ACCESS" }, \
94 { EXIT_REASON_DR_ACCESS, "DR_ACCESS" }, \
95 { EXIT_REASON_IO_INSTRUCTION, "IO_INSTRUCTION" }, \
96 { EXIT_REASON_MSR_READ, "MSR_READ" }, \
97 { EXIT_REASON_MSR_WRITE, "MSR_WRITE" }, \
98 { EXIT_REASON_MWAIT_INSTRUCTION, "MWAIT_INSTRUCTION" }, \
99 { EXIT_REASON_MONITOR_INSTRUCTION, "MONITOR_INSTRUCTION" }, \
100 { EXIT_REASON_PAUSE_INSTRUCTION, "PAUSE_INSTRUCTION" }, \
101 { EXIT_REASON_MCE_DURING_VMENTRY, "MCE_DURING_VMENTRY" }, \
102 { EXIT_REASON_TPR_BELOW_THRESHOLD, "TPR_BELOW_THRESHOLD" }, \
103 { EXIT_REASON_APIC_ACCESS, "APIC_ACCESS" }, \
104 { EXIT_REASON_EPT_VIOLATION, "EPT_VIOLATION" }, \
105 { EXIT_REASON_EPT_MISCONFIG, "EPT_MISCONFIG" }, \
106 { EXIT_REASON_WBINVD, "WBINVD" }
107
108#ifdef __KERNEL__
109
28#include <linux/types.h> 110#include <linux/types.h>
29 111
30/* 112/*
@@ -241,49 +323,6 @@ enum vmcs_field {
241 HOST_RIP = 0x00006c16, 323 HOST_RIP = 0x00006c16,
242}; 324};
243 325
244#define VMX_EXIT_REASONS_FAILED_VMENTRY 0x80000000
245
246#define EXIT_REASON_EXCEPTION_NMI 0
247#define EXIT_REASON_EXTERNAL_INTERRUPT 1
248#define EXIT_REASON_TRIPLE_FAULT 2
249
250#define EXIT_REASON_PENDING_INTERRUPT 7
251#define EXIT_REASON_NMI_WINDOW 8
252#define EXIT_REASON_TASK_SWITCH 9
253#define EXIT_REASON_CPUID 10
254#define EXIT_REASON_HLT 12
255#define EXIT_REASON_INVD 13
256#define EXIT_REASON_INVLPG 14
257#define EXIT_REASON_RDPMC 15
258#define EXIT_REASON_RDTSC 16
259#define EXIT_REASON_VMCALL 18
260#define EXIT_REASON_VMCLEAR 19
261#define EXIT_REASON_VMLAUNCH 20
262#define EXIT_REASON_VMPTRLD 21
263#define EXIT_REASON_VMPTRST 22
264#define EXIT_REASON_VMREAD 23
265#define EXIT_REASON_VMRESUME 24
266#define EXIT_REASON_VMWRITE 25
267#define EXIT_REASON_VMOFF 26
268#define EXIT_REASON_VMON 27
269#define EXIT_REASON_CR_ACCESS 28
270#define EXIT_REASON_DR_ACCESS 29
271#define EXIT_REASON_IO_INSTRUCTION 30
272#define EXIT_REASON_MSR_READ 31
273#define EXIT_REASON_MSR_WRITE 32
274#define EXIT_REASON_INVALID_STATE 33
275#define EXIT_REASON_MWAIT_INSTRUCTION 36
276#define EXIT_REASON_MONITOR_INSTRUCTION 39
277#define EXIT_REASON_PAUSE_INSTRUCTION 40
278#define EXIT_REASON_MCE_DURING_VMENTRY 41
279#define EXIT_REASON_TPR_BELOW_THRESHOLD 43
280#define EXIT_REASON_APIC_ACCESS 44
281#define EXIT_REASON_EPT_VIOLATION 48
282#define EXIT_REASON_EPT_MISCONFIG 49
283#define EXIT_REASON_WBINVD 54
284#define EXIT_REASON_XSETBV 55
285#define EXIT_REASON_INVPCID 58
286
287/* 326/*
288 * Interruption-information format 327 * Interruption-information format
289 */ 328 */
@@ -488,3 +527,5 @@ enum vm_instruction_error_number {
488}; 527};
489 528
490#endif 529#endif
530
531#endif
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 8215e5652d97..8d7a619718b5 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -100,6 +100,8 @@ obj-$(CONFIG_SWIOTLB) += pci-swiotlb.o
100obj-$(CONFIG_OF) += devicetree.o 100obj-$(CONFIG_OF) += devicetree.o
101obj-$(CONFIG_UPROBES) += uprobes.o 101obj-$(CONFIG_UPROBES) += uprobes.o
102 102
103obj-$(CONFIG_PERF_EVENTS) += perf_regs.o
104
103### 105###
104# 64 bit specific files 106# 64 bit specific files
105ifeq ($(CONFIG_X86_64),y) 107ifeq ($(CONFIG_X86_64),y)
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
index 38e4894165b9..db917ec89040 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
@@ -2347,6 +2347,27 @@ int uncore_pmu_event_init(struct perf_event *event)
2347 return ret; 2347 return ret;
2348} 2348}
2349 2349
2350static ssize_t uncore_get_attr_cpumask(struct device *dev,
2351 struct device_attribute *attr, char *buf)
2352{
2353 int n = cpulist_scnprintf(buf, PAGE_SIZE - 2, &uncore_cpu_mask);
2354
2355 buf[n++] = '\n';
2356 buf[n] = '\0';
2357 return n;
2358}
2359
2360static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);
2361
2362static struct attribute *uncore_pmu_attrs[] = {
2363 &dev_attr_cpumask.attr,
2364 NULL,
2365};
2366
2367static struct attribute_group uncore_pmu_attr_group = {
2368 .attrs = uncore_pmu_attrs,
2369};
2370
2350static int __init uncore_pmu_register(struct intel_uncore_pmu *pmu) 2371static int __init uncore_pmu_register(struct intel_uncore_pmu *pmu)
2351{ 2372{
2352 int ret; 2373 int ret;
@@ -2384,8 +2405,8 @@ static void __init uncore_type_exit(struct intel_uncore_type *type)
2384 free_percpu(type->pmus[i].box); 2405 free_percpu(type->pmus[i].box);
2385 kfree(type->pmus); 2406 kfree(type->pmus);
2386 type->pmus = NULL; 2407 type->pmus = NULL;
2387 kfree(type->attr_groups[1]); 2408 kfree(type->events_group);
2388 type->attr_groups[1] = NULL; 2409 type->events_group = NULL;
2389} 2410}
2390 2411
2391static void __init uncore_types_exit(struct intel_uncore_type **types) 2412static void __init uncore_types_exit(struct intel_uncore_type **types)
@@ -2437,9 +2458,10 @@ static int __init uncore_type_init(struct intel_uncore_type *type)
2437 for (j = 0; j < i; j++) 2458 for (j = 0; j < i; j++)
2438 attrs[j] = &type->event_descs[j].attr.attr; 2459 attrs[j] = &type->event_descs[j].attr.attr;
2439 2460
2440 type->attr_groups[1] = events_group; 2461 type->events_group = events_group;
2441 } 2462 }
2442 2463
2464 type->pmu_group = &uncore_pmu_attr_group;
2443 type->pmus = pmus; 2465 type->pmus = pmus;
2444 return 0; 2466 return 0;
2445fail: 2467fail:
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.h b/arch/x86/kernel/cpu/perf_event_intel_uncore.h
index 5b81c1856aac..e68a4550e952 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.h
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.h
@@ -369,10 +369,12 @@ struct intel_uncore_type {
369 struct intel_uncore_pmu *pmus; 369 struct intel_uncore_pmu *pmus;
370 struct intel_uncore_ops *ops; 370 struct intel_uncore_ops *ops;
371 struct uncore_event_desc *event_descs; 371 struct uncore_event_desc *event_descs;
372 const struct attribute_group *attr_groups[3]; 372 const struct attribute_group *attr_groups[4];
373}; 373};
374 374
375#define format_group attr_groups[0] 375#define pmu_group attr_groups[0]
376#define format_group attr_groups[1]
377#define events_group attr_groups[2]
376 378
377struct intel_uncore_ops { 379struct intel_uncore_ops {
378 void (*init_box)(struct intel_uncore_box *); 380 void (*init_box)(struct intel_uncore_box *);
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 623f28837476..f438a44bf8f9 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -1109,17 +1109,21 @@ ENTRY(ftrace_caller)
1109 pushl %eax 1109 pushl %eax
1110 pushl %ecx 1110 pushl %ecx
1111 pushl %edx 1111 pushl %edx
1112 movl 0xc(%esp), %eax 1112 pushl $0 /* Pass NULL as regs pointer */
1113 movl 4*4(%esp), %eax
1113 movl 0x4(%ebp), %edx 1114 movl 0x4(%ebp), %edx
1115 leal function_trace_op, %ecx
1114 subl $MCOUNT_INSN_SIZE, %eax 1116 subl $MCOUNT_INSN_SIZE, %eax
1115 1117
1116.globl ftrace_call 1118.globl ftrace_call
1117ftrace_call: 1119ftrace_call:
1118 call ftrace_stub 1120 call ftrace_stub
1119 1121
1122 addl $4,%esp /* skip NULL pointer */
1120 popl %edx 1123 popl %edx
1121 popl %ecx 1124 popl %ecx
1122 popl %eax 1125 popl %eax
1126ftrace_ret:
1123#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1127#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1124.globl ftrace_graph_call 1128.globl ftrace_graph_call
1125ftrace_graph_call: 1129ftrace_graph_call:
@@ -1131,6 +1135,71 @@ ftrace_stub:
1131 ret 1135 ret
1132END(ftrace_caller) 1136END(ftrace_caller)
1133 1137
1138ENTRY(ftrace_regs_caller)
1139 pushf /* push flags before compare (in cs location) */
1140 cmpl $0, function_trace_stop
1141 jne ftrace_restore_flags
1142
1143 /*
1144 * i386 does not save SS and ESP when coming from kernel.
1145 * Instead, to get sp, &regs->sp is used (see ptrace.h).
1146 * Unfortunately, that means eflags must be at the same location
1147 * as the current return ip is. We move the return ip into the
1148 * ip location, and move flags into the return ip location.
1149 */
1150 pushl 4(%esp) /* save return ip into ip slot */
1151
1152 pushl $0 /* Load 0 into orig_ax */
1153 pushl %gs
1154 pushl %fs
1155 pushl %es
1156 pushl %ds
1157 pushl %eax
1158 pushl %ebp
1159 pushl %edi
1160 pushl %esi
1161 pushl %edx
1162 pushl %ecx
1163 pushl %ebx
1164
1165 movl 13*4(%esp), %eax /* Get the saved flags */
1166 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */
1167 /* clobbering return ip */
1168 movl $__KERNEL_CS,13*4(%esp)
1169
1170 movl 12*4(%esp), %eax /* Load ip (1st parameter) */
1171 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */
1172 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */
1173 leal function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
1174 pushl %esp /* Save pt_regs as 4th parameter */
1175
1176GLOBAL(ftrace_regs_call)
1177 call ftrace_stub
1178
1179 addl $4, %esp /* Skip pt_regs */
1180 movl 14*4(%esp), %eax /* Move flags back into cs */
1181 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */
1182 movl 12*4(%esp), %eax /* Get return ip from regs->ip */
1183 movl %eax, 14*4(%esp) /* Put return ip back for ret */
1184
1185 popl %ebx
1186 popl %ecx
1187 popl %edx
1188 popl %esi
1189 popl %edi
1190 popl %ebp
1191 popl %eax
1192 popl %ds
1193 popl %es
1194 popl %fs
1195 popl %gs
1196 addl $8, %esp /* Skip orig_ax and ip */
1197 popf /* Pop flags at end (no addl to corrupt flags) */
1198 jmp ftrace_ret
1199
1200ftrace_restore_flags:
1201 popf
1202 jmp ftrace_stub
1134#else /* ! CONFIG_DYNAMIC_FTRACE */ 1203#else /* ! CONFIG_DYNAMIC_FTRACE */
1135 1204
1136ENTRY(mcount) 1205ENTRY(mcount)
@@ -1171,9 +1240,6 @@ END(mcount)
1171 1240
1172#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1241#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1173ENTRY(ftrace_graph_caller) 1242ENTRY(ftrace_graph_caller)
1174 cmpl $0, function_trace_stop
1175 jne ftrace_stub
1176
1177 pushl %eax 1243 pushl %eax
1178 pushl %ecx 1244 pushl %ecx
1179 pushl %edx 1245 pushl %edx
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 1a8f3cbb6ee3..4f0322e4ecee 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -69,25 +69,51 @@
69 .section .entry.text, "ax" 69 .section .entry.text, "ax"
70 70
71#ifdef CONFIG_FUNCTION_TRACER 71#ifdef CONFIG_FUNCTION_TRACER
72
73#ifdef CC_USING_FENTRY
74# define function_hook __fentry__
75#else
76# define function_hook mcount
77#endif
78
72#ifdef CONFIG_DYNAMIC_FTRACE 79#ifdef CONFIG_DYNAMIC_FTRACE
73ENTRY(mcount) 80
81ENTRY(function_hook)
74 retq 82 retq
75END(mcount) 83END(function_hook)
84
85/* skip is set if stack has been adjusted */
86.macro ftrace_caller_setup skip=0
87 MCOUNT_SAVE_FRAME \skip
88
89 /* Load the ftrace_ops into the 3rd parameter */
90 leaq function_trace_op, %rdx
91
92 /* Load ip into the first parameter */
93 movq RIP(%rsp), %rdi
94 subq $MCOUNT_INSN_SIZE, %rdi
95 /* Load the parent_ip into the second parameter */
96#ifdef CC_USING_FENTRY
97 movq SS+16(%rsp), %rsi
98#else
99 movq 8(%rbp), %rsi
100#endif
101.endm
76 102
77ENTRY(ftrace_caller) 103ENTRY(ftrace_caller)
104 /* Check if tracing was disabled (quick check) */
78 cmpl $0, function_trace_stop 105 cmpl $0, function_trace_stop
79 jne ftrace_stub 106 jne ftrace_stub
80 107
81 MCOUNT_SAVE_FRAME 108 ftrace_caller_setup
82 109 /* regs go into 4th parameter (but make it NULL) */
83 movq 0x38(%rsp), %rdi 110 movq $0, %rcx
84 movq 8(%rbp), %rsi
85 subq $MCOUNT_INSN_SIZE, %rdi
86 111
87GLOBAL(ftrace_call) 112GLOBAL(ftrace_call)
88 call ftrace_stub 113 call ftrace_stub
89 114
90 MCOUNT_RESTORE_FRAME 115 MCOUNT_RESTORE_FRAME
116ftrace_return:
91 117
92#ifdef CONFIG_FUNCTION_GRAPH_TRACER 118#ifdef CONFIG_FUNCTION_GRAPH_TRACER
93GLOBAL(ftrace_graph_call) 119GLOBAL(ftrace_graph_call)
@@ -98,8 +124,78 @@ GLOBAL(ftrace_stub)
98 retq 124 retq
99END(ftrace_caller) 125END(ftrace_caller)
100 126
127ENTRY(ftrace_regs_caller)
128 /* Save the current flags before compare (in SS location)*/
129 pushfq
130
131 /* Check if tracing was disabled (quick check) */
132 cmpl $0, function_trace_stop
133 jne ftrace_restore_flags
134
135 /* skip=8 to skip flags saved in SS */
136 ftrace_caller_setup 8
137
138 /* Save the rest of pt_regs */
139 movq %r15, R15(%rsp)
140 movq %r14, R14(%rsp)
141 movq %r13, R13(%rsp)
142 movq %r12, R12(%rsp)
143 movq %r11, R11(%rsp)
144 movq %r10, R10(%rsp)
145 movq %rbp, RBP(%rsp)
146 movq %rbx, RBX(%rsp)
147 /* Copy saved flags */
148 movq SS(%rsp), %rcx
149 movq %rcx, EFLAGS(%rsp)
150 /* Kernel segments */
151 movq $__KERNEL_DS, %rcx
152 movq %rcx, SS(%rsp)
153 movq $__KERNEL_CS, %rcx
154 movq %rcx, CS(%rsp)
155 /* Stack - skipping return address */
156 leaq SS+16(%rsp), %rcx
157 movq %rcx, RSP(%rsp)
158
159 /* regs go into 4th parameter */
160 leaq (%rsp), %rcx
161
162GLOBAL(ftrace_regs_call)
163 call ftrace_stub
164
165 /* Copy flags back to SS, to restore them */
166 movq EFLAGS(%rsp), %rax
167 movq %rax, SS(%rsp)
168
169 /* Handlers can change the RIP */
170 movq RIP(%rsp), %rax
171 movq %rax, SS+8(%rsp)
172
173 /* restore the rest of pt_regs */
174 movq R15(%rsp), %r15
175 movq R14(%rsp), %r14
176 movq R13(%rsp), %r13
177 movq R12(%rsp), %r12
178 movq R10(%rsp), %r10
179 movq RBP(%rsp), %rbp
180 movq RBX(%rsp), %rbx
181
182 /* skip=8 to skip flags saved in SS */
183 MCOUNT_RESTORE_FRAME 8
184
185 /* Restore flags */
186 popfq
187
188 jmp ftrace_return
189ftrace_restore_flags:
190 popfq
191 jmp ftrace_stub
192
193END(ftrace_regs_caller)
194
195
101#else /* ! CONFIG_DYNAMIC_FTRACE */ 196#else /* ! CONFIG_DYNAMIC_FTRACE */
102ENTRY(mcount) 197
198ENTRY(function_hook)
103 cmpl $0, function_trace_stop 199 cmpl $0, function_trace_stop
104 jne ftrace_stub 200 jne ftrace_stub
105 201
@@ -120,8 +216,12 @@ GLOBAL(ftrace_stub)
120trace: 216trace:
121 MCOUNT_SAVE_FRAME 217 MCOUNT_SAVE_FRAME
122 218
123 movq 0x38(%rsp), %rdi 219 movq RIP(%rsp), %rdi
220#ifdef CC_USING_FENTRY
221 movq SS+16(%rsp), %rsi
222#else
124 movq 8(%rbp), %rsi 223 movq 8(%rbp), %rsi
224#endif
125 subq $MCOUNT_INSN_SIZE, %rdi 225 subq $MCOUNT_INSN_SIZE, %rdi
126 226
127 call *ftrace_trace_function 227 call *ftrace_trace_function
@@ -129,20 +229,22 @@ trace:
129 MCOUNT_RESTORE_FRAME 229 MCOUNT_RESTORE_FRAME
130 230
131 jmp ftrace_stub 231 jmp ftrace_stub
132END(mcount) 232END(function_hook)
133#endif /* CONFIG_DYNAMIC_FTRACE */ 233#endif /* CONFIG_DYNAMIC_FTRACE */
134#endif /* CONFIG_FUNCTION_TRACER */ 234#endif /* CONFIG_FUNCTION_TRACER */
135 235
136#ifdef CONFIG_FUNCTION_GRAPH_TRACER 236#ifdef CONFIG_FUNCTION_GRAPH_TRACER
137ENTRY(ftrace_graph_caller) 237ENTRY(ftrace_graph_caller)
138 cmpl $0, function_trace_stop
139 jne ftrace_stub
140
141 MCOUNT_SAVE_FRAME 238 MCOUNT_SAVE_FRAME
142 239
240#ifdef CC_USING_FENTRY
241 leaq SS+16(%rsp), %rdi
242 movq $0, %rdx /* No framepointers needed */
243#else
143 leaq 8(%rbp), %rdi 244 leaq 8(%rbp), %rdi
144 movq 0x38(%rsp), %rsi
145 movq (%rbp), %rdx 245 movq (%rbp), %rdx
246#endif
247 movq RIP(%rsp), %rsi
146 subq $MCOUNT_INSN_SIZE, %rsi 248 subq $MCOUNT_INSN_SIZE, %rsi
147 249
148 call prepare_ftrace_return 250 call prepare_ftrace_return
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index c3a7cb4bf6e6..1d414029f1d8 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -206,6 +206,21 @@ static int
206ftrace_modify_code(unsigned long ip, unsigned const char *old_code, 206ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
207 unsigned const char *new_code); 207 unsigned const char *new_code);
208 208
209/*
210 * Should never be called:
211 * As it is only called by __ftrace_replace_code() which is called by
212 * ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
213 * which is called to turn mcount into nops or nops into function calls
214 * but not to convert a function from not using regs to one that uses
215 * regs, which ftrace_modify_call() is for.
216 */
217int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
218 unsigned long addr)
219{
220 WARN_ON(1);
221 return -EINVAL;
222}
223
209int ftrace_update_ftrace_func(ftrace_func_t func) 224int ftrace_update_ftrace_func(ftrace_func_t func)
210{ 225{
211 unsigned long ip = (unsigned long)(&ftrace_call); 226 unsigned long ip = (unsigned long)(&ftrace_call);
@@ -220,6 +235,14 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
220 235
221 ret = ftrace_modify_code(ip, old, new); 236 ret = ftrace_modify_code(ip, old, new);
222 237
238 /* Also update the regs callback function */
239 if (!ret) {
240 ip = (unsigned long)(&ftrace_regs_call);
241 memcpy(old, &ftrace_regs_call, MCOUNT_INSN_SIZE);
242 new = ftrace_call_replace(ip, (unsigned long)func);
243 ret = ftrace_modify_code(ip, old, new);
244 }
245
223 atomic_dec(&modifying_ftrace_code); 246 atomic_dec(&modifying_ftrace_code);
224 247
225 return ret; 248 return ret;
@@ -299,6 +322,32 @@ static int add_brk_on_nop(struct dyn_ftrace *rec)
299 return add_break(rec->ip, old); 322 return add_break(rec->ip, old);
300} 323}
301 324
325/*
326 * If the record has the FTRACE_FL_REGS set, that means that it
327 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
328 * is not not set, then it wants to convert to the normal callback.
329 */
330static unsigned long get_ftrace_addr(struct dyn_ftrace *rec)
331{
332 if (rec->flags & FTRACE_FL_REGS)
333 return (unsigned long)FTRACE_REGS_ADDR;
334 else
335 return (unsigned long)FTRACE_ADDR;
336}
337
338/*
339 * The FTRACE_FL_REGS_EN is set when the record already points to
340 * a function that saves all the regs. Basically the '_EN' version
341 * represents the current state of the function.
342 */
343static unsigned long get_ftrace_old_addr(struct dyn_ftrace *rec)
344{
345 if (rec->flags & FTRACE_FL_REGS_EN)
346 return (unsigned long)FTRACE_REGS_ADDR;
347 else
348 return (unsigned long)FTRACE_ADDR;
349}
350
302static int add_breakpoints(struct dyn_ftrace *rec, int enable) 351static int add_breakpoints(struct dyn_ftrace *rec, int enable)
303{ 352{
304 unsigned long ftrace_addr; 353 unsigned long ftrace_addr;
@@ -306,7 +355,7 @@ static int add_breakpoints(struct dyn_ftrace *rec, int enable)
306 355
307 ret = ftrace_test_record(rec, enable); 356 ret = ftrace_test_record(rec, enable);
308 357
309 ftrace_addr = (unsigned long)FTRACE_ADDR; 358 ftrace_addr = get_ftrace_addr(rec);
310 359
311 switch (ret) { 360 switch (ret) {
312 case FTRACE_UPDATE_IGNORE: 361 case FTRACE_UPDATE_IGNORE:
@@ -316,6 +365,10 @@ static int add_breakpoints(struct dyn_ftrace *rec, int enable)
316 /* converting nop to call */ 365 /* converting nop to call */
317 return add_brk_on_nop(rec); 366 return add_brk_on_nop(rec);
318 367
368 case FTRACE_UPDATE_MODIFY_CALL_REGS:
369 case FTRACE_UPDATE_MODIFY_CALL:
370 ftrace_addr = get_ftrace_old_addr(rec);
371 /* fall through */
319 case FTRACE_UPDATE_MAKE_NOP: 372 case FTRACE_UPDATE_MAKE_NOP:
320 /* converting a call to a nop */ 373 /* converting a call to a nop */
321 return add_brk_on_call(rec, ftrace_addr); 374 return add_brk_on_call(rec, ftrace_addr);
@@ -360,13 +413,21 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
360 * If not, don't touch the breakpoint, we make just create 413 * If not, don't touch the breakpoint, we make just create
361 * a disaster. 414 * a disaster.
362 */ 415 */
363 ftrace_addr = (unsigned long)FTRACE_ADDR; 416 ftrace_addr = get_ftrace_addr(rec);
417 nop = ftrace_call_replace(ip, ftrace_addr);
418
419 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
420 goto update;
421
422 /* Check both ftrace_addr and ftrace_old_addr */
423 ftrace_addr = get_ftrace_old_addr(rec);
364 nop = ftrace_call_replace(ip, ftrace_addr); 424 nop = ftrace_call_replace(ip, ftrace_addr);
365 425
366 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) 426 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
367 return -EINVAL; 427 return -EINVAL;
368 } 428 }
369 429
430 update:
370 return probe_kernel_write((void *)ip, &nop[0], 1); 431 return probe_kernel_write((void *)ip, &nop[0], 1);
371} 432}
372 433
@@ -405,12 +466,14 @@ static int add_update(struct dyn_ftrace *rec, int enable)
405 466
406 ret = ftrace_test_record(rec, enable); 467 ret = ftrace_test_record(rec, enable);
407 468
408 ftrace_addr = (unsigned long)FTRACE_ADDR; 469 ftrace_addr = get_ftrace_addr(rec);
409 470
410 switch (ret) { 471 switch (ret) {
411 case FTRACE_UPDATE_IGNORE: 472 case FTRACE_UPDATE_IGNORE:
412 return 0; 473 return 0;
413 474
475 case FTRACE_UPDATE_MODIFY_CALL_REGS:
476 case FTRACE_UPDATE_MODIFY_CALL:
414 case FTRACE_UPDATE_MAKE_CALL: 477 case FTRACE_UPDATE_MAKE_CALL:
415 /* converting nop to call */ 478 /* converting nop to call */
416 return add_update_call(rec, ftrace_addr); 479 return add_update_call(rec, ftrace_addr);
@@ -455,12 +518,14 @@ static int finish_update(struct dyn_ftrace *rec, int enable)
455 518
456 ret = ftrace_update_record(rec, enable); 519 ret = ftrace_update_record(rec, enable);
457 520
458 ftrace_addr = (unsigned long)FTRACE_ADDR; 521 ftrace_addr = get_ftrace_addr(rec);
459 522
460 switch (ret) { 523 switch (ret) {
461 case FTRACE_UPDATE_IGNORE: 524 case FTRACE_UPDATE_IGNORE:
462 return 0; 525 return 0;
463 526
527 case FTRACE_UPDATE_MODIFY_CALL_REGS:
528 case FTRACE_UPDATE_MODIFY_CALL:
464 case FTRACE_UPDATE_MAKE_CALL: 529 case FTRACE_UPDATE_MAKE_CALL:
465 /* converting nop to call */ 530 /* converting nop to call */
466 return finish_update_call(rec, ftrace_addr); 531 return finish_update_call(rec, ftrace_addr);
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index e2f751efb7b1..57916c0d3cf6 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -541,6 +541,23 @@ reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb
541 return 1; 541 return 1;
542} 542}
543 543
544#ifdef KPROBES_CAN_USE_FTRACE
545static void __kprobes skip_singlestep(struct kprobe *p, struct pt_regs *regs,
546 struct kprobe_ctlblk *kcb)
547{
548 /*
549 * Emulate singlestep (and also recover regs->ip)
550 * as if there is a 5byte nop
551 */
552 regs->ip = (unsigned long)p->addr + MCOUNT_INSN_SIZE;
553 if (unlikely(p->post_handler)) {
554 kcb->kprobe_status = KPROBE_HIT_SSDONE;
555 p->post_handler(p, regs, 0);
556 }
557 __this_cpu_write(current_kprobe, NULL);
558}
559#endif
560
544/* 561/*
545 * Interrupts are disabled on entry as trap3 is an interrupt gate and they 562 * Interrupts are disabled on entry as trap3 is an interrupt gate and they
546 * remain disabled throughout this function. 563 * remain disabled throughout this function.
@@ -599,6 +616,12 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
599 } else if (kprobe_running()) { 616 } else if (kprobe_running()) {
600 p = __this_cpu_read(current_kprobe); 617 p = __this_cpu_read(current_kprobe);
601 if (p->break_handler && p->break_handler(p, regs)) { 618 if (p->break_handler && p->break_handler(p, regs)) {
619#ifdef KPROBES_CAN_USE_FTRACE
620 if (kprobe_ftrace(p)) {
621 skip_singlestep(p, regs, kcb);
622 return 1;
623 }
624#endif
602 setup_singlestep(p, regs, kcb, 0); 625 setup_singlestep(p, regs, kcb, 0);
603 return 1; 626 return 1;
604 } 627 }
@@ -1052,6 +1075,50 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
1052 return 0; 1075 return 0;
1053} 1076}
1054 1077
1078#ifdef KPROBES_CAN_USE_FTRACE
1079/* Ftrace callback handler for kprobes */
1080void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
1081 struct ftrace_ops *ops, struct pt_regs *regs)
1082{
1083 struct kprobe *p;
1084 struct kprobe_ctlblk *kcb;
1085 unsigned long flags;
1086
1087 /* Disable irq for emulating a breakpoint and avoiding preempt */
1088 local_irq_save(flags);
1089
1090 p = get_kprobe((kprobe_opcode_t *)ip);
1091 if (unlikely(!p) || kprobe_disabled(p))
1092 goto end;
1093
1094 kcb = get_kprobe_ctlblk();
1095 if (kprobe_running()) {
1096 kprobes_inc_nmissed_count(p);
1097 } else {
1098 /* Kprobe handler expects regs->ip = ip + 1 as breakpoint hit */
1099 regs->ip = ip + sizeof(kprobe_opcode_t);
1100
1101 __this_cpu_write(current_kprobe, p);
1102 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1103 if (!p->pre_handler || !p->pre_handler(p, regs))
1104 skip_singlestep(p, regs, kcb);
1105 /*
1106 * If pre_handler returns !0, it sets regs->ip and
1107 * resets current kprobe.
1108 */
1109 }
1110end:
1111 local_irq_restore(flags);
1112}
1113
1114int __kprobes arch_prepare_kprobe_ftrace(struct kprobe *p)
1115{
1116 p->ainsn.insn = NULL;
1117 p->ainsn.boostable = -1;
1118 return 0;
1119}
1120#endif
1121
1055int __init arch_init_kprobes(void) 1122int __init arch_init_kprobes(void)
1056{ 1123{
1057 return arch_init_optprobes(); 1124 return arch_init_optprobes();
diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c
new file mode 100644
index 000000000000..e309cc5c276e
--- /dev/null
+++ b/arch/x86/kernel/perf_regs.c
@@ -0,0 +1,105 @@
1#include <linux/errno.h>
2#include <linux/kernel.h>
3#include <linux/sched.h>
4#include <linux/perf_event.h>
5#include <linux/bug.h>
6#include <linux/stddef.h>
7#include <asm/perf_regs.h>
8#include <asm/ptrace.h>
9
10#ifdef CONFIG_X86_32
11#define PERF_REG_X86_MAX PERF_REG_X86_32_MAX
12#else
13#define PERF_REG_X86_MAX PERF_REG_X86_64_MAX
14#endif
15
16#define PT_REGS_OFFSET(id, r) [id] = offsetof(struct pt_regs, r)
17
18static unsigned int pt_regs_offset[PERF_REG_X86_MAX] = {
19 PT_REGS_OFFSET(PERF_REG_X86_AX, ax),
20 PT_REGS_OFFSET(PERF_REG_X86_BX, bx),
21 PT_REGS_OFFSET(PERF_REG_X86_CX, cx),
22 PT_REGS_OFFSET(PERF_REG_X86_DX, dx),
23 PT_REGS_OFFSET(PERF_REG_X86_SI, si),
24 PT_REGS_OFFSET(PERF_REG_X86_DI, di),
25 PT_REGS_OFFSET(PERF_REG_X86_BP, bp),
26 PT_REGS_OFFSET(PERF_REG_X86_SP, sp),
27 PT_REGS_OFFSET(PERF_REG_X86_IP, ip),
28 PT_REGS_OFFSET(PERF_REG_X86_FLAGS, flags),
29 PT_REGS_OFFSET(PERF_REG_X86_CS, cs),
30 PT_REGS_OFFSET(PERF_REG_X86_SS, ss),
31#ifdef CONFIG_X86_32
32 PT_REGS_OFFSET(PERF_REG_X86_DS, ds),
33 PT_REGS_OFFSET(PERF_REG_X86_ES, es),
34 PT_REGS_OFFSET(PERF_REG_X86_FS, fs),
35 PT_REGS_OFFSET(PERF_REG_X86_GS, gs),
36#else
37 /*
38 * The pt_regs struct does not store
39 * ds, es, fs, gs in 64 bit mode.
40 */
41 (unsigned int) -1,
42 (unsigned int) -1,
43 (unsigned int) -1,
44 (unsigned int) -1,
45#endif
46#ifdef CONFIG_X86_64
47 PT_REGS_OFFSET(PERF_REG_X86_R8, r8),
48 PT_REGS_OFFSET(PERF_REG_X86_R9, r9),
49 PT_REGS_OFFSET(PERF_REG_X86_R10, r10),
50 PT_REGS_OFFSET(PERF_REG_X86_R11, r11),
51 PT_REGS_OFFSET(PERF_REG_X86_R12, r12),
52 PT_REGS_OFFSET(PERF_REG_X86_R13, r13),
53 PT_REGS_OFFSET(PERF_REG_X86_R14, r14),
54 PT_REGS_OFFSET(PERF_REG_X86_R15, r15),
55#endif
56};
57
58u64 perf_reg_value(struct pt_regs *regs, int idx)
59{
60 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset)))
61 return 0;
62
63 return regs_get_register(regs, pt_regs_offset[idx]);
64}
65
66#define REG_RESERVED (~((1ULL << PERF_REG_X86_MAX) - 1ULL))
67
68#ifdef CONFIG_X86_32
69int perf_reg_validate(u64 mask)
70{
71 if (!mask || mask & REG_RESERVED)
72 return -EINVAL;
73
74 return 0;
75}
76
77u64 perf_reg_abi(struct task_struct *task)
78{
79 return PERF_SAMPLE_REGS_ABI_32;
80}
81#else /* CONFIG_X86_64 */
82#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \
83 (1ULL << PERF_REG_X86_ES) | \
84 (1ULL << PERF_REG_X86_FS) | \
85 (1ULL << PERF_REG_X86_GS))
86
87int perf_reg_validate(u64 mask)
88{
89 if (!mask || mask & REG_RESERVED)
90 return -EINVAL;
91
92 if (mask & REG_NOSUPPORT)
93 return -EINVAL;
94
95 return 0;
96}
97
98u64 perf_reg_abi(struct task_struct *task)
99{
100 if (test_tsk_thread_flag(task, TIF_IA32))
101 return PERF_SAMPLE_REGS_ABI_32;
102 else
103 return PERF_SAMPLE_REGS_ABI_64;
104}
105#endif /* CONFIG_X86_32 */
diff --git a/arch/x86/kernel/step.c b/arch/x86/kernel/step.c
index c346d1161488..cd3b2438a980 100644
--- a/arch/x86/kernel/step.c
+++ b/arch/x86/kernel/step.c
@@ -157,6 +157,33 @@ static int enable_single_step(struct task_struct *child)
157 return 1; 157 return 1;
158} 158}
159 159
160void set_task_blockstep(struct task_struct *task, bool on)
161{
162 unsigned long debugctl;
163
164 /*
165 * Ensure irq/preemption can't change debugctl in between.
166 * Note also that both TIF_BLOCKSTEP and debugctl should
167 * be changed atomically wrt preemption.
168 * FIXME: this means that set/clear TIF_BLOCKSTEP is simply
169 * wrong if task != current, SIGKILL can wakeup the stopped
170 * tracee and set/clear can play with the running task, this
171 * can confuse the next __switch_to_xtra().
172 */
173 local_irq_disable();
174 debugctl = get_debugctlmsr();
175 if (on) {
176 debugctl |= DEBUGCTLMSR_BTF;
177 set_tsk_thread_flag(task, TIF_BLOCKSTEP);
178 } else {
179 debugctl &= ~DEBUGCTLMSR_BTF;
180 clear_tsk_thread_flag(task, TIF_BLOCKSTEP);
181 }
182 if (task == current)
183 update_debugctlmsr(debugctl);
184 local_irq_enable();
185}
186
160/* 187/*
161 * Enable single or block step. 188 * Enable single or block step.
162 */ 189 */
@@ -169,19 +196,10 @@ static void enable_step(struct task_struct *child, bool block)
169 * So no one should try to use debugger block stepping in a program 196 * So no one should try to use debugger block stepping in a program
170 * that uses user-mode single stepping itself. 197 * that uses user-mode single stepping itself.
171 */ 198 */
172 if (enable_single_step(child) && block) { 199 if (enable_single_step(child) && block)
173 unsigned long debugctl = get_debugctlmsr(); 200 set_task_blockstep(child, true);
174 201 else if (test_tsk_thread_flag(child, TIF_BLOCKSTEP))
175 debugctl |= DEBUGCTLMSR_BTF; 202 set_task_blockstep(child, false);
176 update_debugctlmsr(debugctl);
177 set_tsk_thread_flag(child, TIF_BLOCKSTEP);
178 } else if (test_tsk_thread_flag(child, TIF_BLOCKSTEP)) {
179 unsigned long debugctl = get_debugctlmsr();
180
181 debugctl &= ~DEBUGCTLMSR_BTF;
182 update_debugctlmsr(debugctl);
183 clear_tsk_thread_flag(child, TIF_BLOCKSTEP);
184 }
185} 203}
186 204
187void user_enable_single_step(struct task_struct *child) 205void user_enable_single_step(struct task_struct *child)
@@ -199,13 +217,8 @@ void user_disable_single_step(struct task_struct *child)
199 /* 217 /*
200 * Make sure block stepping (BTF) is disabled. 218 * Make sure block stepping (BTF) is disabled.
201 */ 219 */
202 if (test_tsk_thread_flag(child, TIF_BLOCKSTEP)) { 220 if (test_tsk_thread_flag(child, TIF_BLOCKSTEP))
203 unsigned long debugctl = get_debugctlmsr(); 221 set_task_blockstep(child, false);
204
205 debugctl &= ~DEBUGCTLMSR_BTF;
206 update_debugctlmsr(debugctl);
207 clear_tsk_thread_flag(child, TIF_BLOCKSTEP);
208 }
209 222
210 /* Always clear TIF_SINGLESTEP... */ 223 /* Always clear TIF_SINGLESTEP... */
211 clear_tsk_thread_flag(child, TIF_SINGLESTEP); 224 clear_tsk_thread_flag(child, TIF_SINGLESTEP);
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index 36fd42091fa7..9538f00827a9 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -41,6 +41,9 @@
41/* Adjust the return address of a call insn */ 41/* Adjust the return address of a call insn */
42#define UPROBE_FIX_CALL 0x2 42#define UPROBE_FIX_CALL 0x2
43 43
44/* Instruction will modify TF, don't change it */
45#define UPROBE_FIX_SETF 0x4
46
44#define UPROBE_FIX_RIP_AX 0x8000 47#define UPROBE_FIX_RIP_AX 0x8000
45#define UPROBE_FIX_RIP_CX 0x4000 48#define UPROBE_FIX_RIP_CX 0x4000
46 49
@@ -239,6 +242,10 @@ static void prepare_fixups(struct arch_uprobe *auprobe, struct insn *insn)
239 insn_get_opcode(insn); /* should be a nop */ 242 insn_get_opcode(insn); /* should be a nop */
240 243
241 switch (OPCODE1(insn)) { 244 switch (OPCODE1(insn)) {
245 case 0x9d:
246 /* popf */
247 auprobe->fixups |= UPROBE_FIX_SETF;
248 break;
242 case 0xc3: /* ret/lret */ 249 case 0xc3: /* ret/lret */
243 case 0xcb: 250 case 0xcb:
244 case 0xc2: 251 case 0xc2:
@@ -646,7 +653,7 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
646 * Skip these instructions as per the currently known x86 ISA. 653 * Skip these instructions as per the currently known x86 ISA.
647 * 0x66* { 0x90 | 0x0f 0x1f | 0x0f 0x19 | 0x87 0xc0 } 654 * 0x66* { 0x90 | 0x0f 0x1f | 0x0f 0x19 | 0x87 0xc0 }
648 */ 655 */
649bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) 656static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
650{ 657{
651 int i; 658 int i;
652 659
@@ -673,3 +680,46 @@ bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
673 } 680 }
674 return false; 681 return false;
675} 682}
683
684bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
685{
686 bool ret = __skip_sstep(auprobe, regs);
687 if (ret && (regs->flags & X86_EFLAGS_TF))
688 send_sig(SIGTRAP, current, 0);
689 return ret;
690}
691
692void arch_uprobe_enable_step(struct arch_uprobe *auprobe)
693{
694 struct task_struct *task = current;
695 struct arch_uprobe_task *autask = &task->utask->autask;
696 struct pt_regs *regs = task_pt_regs(task);
697
698 autask->saved_tf = !!(regs->flags & X86_EFLAGS_TF);
699
700 regs->flags |= X86_EFLAGS_TF;
701 if (test_tsk_thread_flag(task, TIF_BLOCKSTEP))
702 set_task_blockstep(task, false);
703}
704
705void arch_uprobe_disable_step(struct arch_uprobe *auprobe)
706{
707 struct task_struct *task = current;
708 struct arch_uprobe_task *autask = &task->utask->autask;
709 bool trapped = (task->utask->state == UTASK_SSTEP_TRAPPED);
710 struct pt_regs *regs = task_pt_regs(task);
711 /*
712 * The state of TIF_BLOCKSTEP was not saved so we can get an extra
713 * SIGTRAP if we do not clear TF. We need to examine the opcode to
714 * make it right.
715 */
716 if (unlikely(trapped)) {
717 if (!autask->saved_tf)
718 regs->flags &= ~X86_EFLAGS_TF;
719 } else {
720 if (autask->saved_tf)
721 send_sig(SIGTRAP, task, 0);
722 else if (!(auprobe->fixups & UPROBE_FIX_SETF))
723 regs->flags &= ~X86_EFLAGS_TF;
724 }
725}
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index 6020f6f5927c..1330dd102950 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -13,9 +13,13 @@
13#include <asm/ftrace.h> 13#include <asm/ftrace.h>
14 14
15#ifdef CONFIG_FUNCTION_TRACER 15#ifdef CONFIG_FUNCTION_TRACER
16/* mcount is defined in assembly */ 16/* mcount and __fentry__ are defined in assembly */
17#ifdef CC_USING_FENTRY
18EXPORT_SYMBOL(__fentry__);
19#else
17EXPORT_SYMBOL(mcount); 20EXPORT_SYMBOL(mcount);
18#endif 21#endif
22#endif
19 23
20EXPORT_SYMBOL(__get_user_1); 24EXPORT_SYMBOL(__get_user_1);
21EXPORT_SYMBOL(__get_user_2); 25EXPORT_SYMBOL(__get_user_2);
diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
index a71faf727ff3..bca63f04dccb 100644
--- a/arch/x86/kvm/trace.h
+++ b/arch/x86/kvm/trace.h
@@ -183,95 +183,6 @@ TRACE_EVENT(kvm_apic,
183#define KVM_ISA_VMX 1 183#define KVM_ISA_VMX 1
184#define KVM_ISA_SVM 2 184#define KVM_ISA_SVM 2
185 185
186#define VMX_EXIT_REASONS \
187 { EXIT_REASON_EXCEPTION_NMI, "EXCEPTION_NMI" }, \
188 { EXIT_REASON_EXTERNAL_INTERRUPT, "EXTERNAL_INTERRUPT" }, \
189 { EXIT_REASON_TRIPLE_FAULT, "TRIPLE_FAULT" }, \
190 { EXIT_REASON_PENDING_INTERRUPT, "PENDING_INTERRUPT" }, \
191 { EXIT_REASON_NMI_WINDOW, "NMI_WINDOW" }, \
192 { EXIT_REASON_TASK_SWITCH, "TASK_SWITCH" }, \
193 { EXIT_REASON_CPUID, "CPUID" }, \
194 { EXIT_REASON_HLT, "HLT" }, \
195 { EXIT_REASON_INVLPG, "INVLPG" }, \
196 { EXIT_REASON_RDPMC, "RDPMC" }, \
197 { EXIT_REASON_RDTSC, "RDTSC" }, \
198 { EXIT_REASON_VMCALL, "VMCALL" }, \
199 { EXIT_REASON_VMCLEAR, "VMCLEAR" }, \
200 { EXIT_REASON_VMLAUNCH, "VMLAUNCH" }, \
201 { EXIT_REASON_VMPTRLD, "VMPTRLD" }, \
202 { EXIT_REASON_VMPTRST, "VMPTRST" }, \
203 { EXIT_REASON_VMREAD, "VMREAD" }, \
204 { EXIT_REASON_VMRESUME, "VMRESUME" }, \
205 { EXIT_REASON_VMWRITE, "VMWRITE" }, \
206 { EXIT_REASON_VMOFF, "VMOFF" }, \
207 { EXIT_REASON_VMON, "VMON" }, \
208 { EXIT_REASON_CR_ACCESS, "CR_ACCESS" }, \
209 { EXIT_REASON_DR_ACCESS, "DR_ACCESS" }, \
210 { EXIT_REASON_IO_INSTRUCTION, "IO_INSTRUCTION" }, \
211 { EXIT_REASON_MSR_READ, "MSR_READ" }, \
212 { EXIT_REASON_MSR_WRITE, "MSR_WRITE" }, \
213 { EXIT_REASON_MWAIT_INSTRUCTION, "MWAIT_INSTRUCTION" }, \
214 { EXIT_REASON_MONITOR_INSTRUCTION, "MONITOR_INSTRUCTION" }, \
215 { EXIT_REASON_PAUSE_INSTRUCTION, "PAUSE_INSTRUCTION" }, \
216 { EXIT_REASON_MCE_DURING_VMENTRY, "MCE_DURING_VMENTRY" }, \
217 { EXIT_REASON_TPR_BELOW_THRESHOLD, "TPR_BELOW_THRESHOLD" }, \
218 { EXIT_REASON_APIC_ACCESS, "APIC_ACCESS" }, \
219 { EXIT_REASON_EPT_VIOLATION, "EPT_VIOLATION" }, \
220 { EXIT_REASON_EPT_MISCONFIG, "EPT_MISCONFIG" }, \
221 { EXIT_REASON_WBINVD, "WBINVD" }
222
223#define SVM_EXIT_REASONS \
224 { SVM_EXIT_READ_CR0, "read_cr0" }, \
225 { SVM_EXIT_READ_CR3, "read_cr3" }, \
226 { SVM_EXIT_READ_CR4, "read_cr4" }, \
227 { SVM_EXIT_READ_CR8, "read_cr8" }, \
228 { SVM_EXIT_WRITE_CR0, "write_cr0" }, \
229 { SVM_EXIT_WRITE_CR3, "write_cr3" }, \
230 { SVM_EXIT_WRITE_CR4, "write_cr4" }, \
231 { SVM_EXIT_WRITE_CR8, "write_cr8" }, \
232 { SVM_EXIT_READ_DR0, "read_dr0" }, \
233 { SVM_EXIT_READ_DR1, "read_dr1" }, \
234 { SVM_EXIT_READ_DR2, "read_dr2" }, \
235 { SVM_EXIT_READ_DR3, "read_dr3" }, \
236 { SVM_EXIT_WRITE_DR0, "write_dr0" }, \
237 { SVM_EXIT_WRITE_DR1, "write_dr1" }, \
238 { SVM_EXIT_WRITE_DR2, "write_dr2" }, \
239 { SVM_EXIT_WRITE_DR3, "write_dr3" }, \
240 { SVM_EXIT_WRITE_DR5, "write_dr5" }, \
241 { SVM_EXIT_WRITE_DR7, "write_dr7" }, \
242 { SVM_EXIT_EXCP_BASE + DB_VECTOR, "DB excp" }, \
243 { SVM_EXIT_EXCP_BASE + BP_VECTOR, "BP excp" }, \
244 { SVM_EXIT_EXCP_BASE + UD_VECTOR, "UD excp" }, \
245 { SVM_EXIT_EXCP_BASE + PF_VECTOR, "PF excp" }, \
246 { SVM_EXIT_EXCP_BASE + NM_VECTOR, "NM excp" }, \
247 { SVM_EXIT_EXCP_BASE + MC_VECTOR, "MC excp" }, \
248 { SVM_EXIT_INTR, "interrupt" }, \
249 { SVM_EXIT_NMI, "nmi" }, \
250 { SVM_EXIT_SMI, "smi" }, \
251 { SVM_EXIT_INIT, "init" }, \
252 { SVM_EXIT_VINTR, "vintr" }, \
253 { SVM_EXIT_CPUID, "cpuid" }, \
254 { SVM_EXIT_INVD, "invd" }, \
255 { SVM_EXIT_HLT, "hlt" }, \
256 { SVM_EXIT_INVLPG, "invlpg" }, \
257 { SVM_EXIT_INVLPGA, "invlpga" }, \
258 { SVM_EXIT_IOIO, "io" }, \
259 { SVM_EXIT_MSR, "msr" }, \
260 { SVM_EXIT_TASK_SWITCH, "task_switch" }, \
261 { SVM_EXIT_SHUTDOWN, "shutdown" }, \
262 { SVM_EXIT_VMRUN, "vmrun" }, \
263 { SVM_EXIT_VMMCALL, "hypercall" }, \
264 { SVM_EXIT_VMLOAD, "vmload" }, \
265 { SVM_EXIT_VMSAVE, "vmsave" }, \
266 { SVM_EXIT_STGI, "stgi" }, \
267 { SVM_EXIT_CLGI, "clgi" }, \
268 { SVM_EXIT_SKINIT, "skinit" }, \
269 { SVM_EXIT_WBINVD, "wbinvd" }, \
270 { SVM_EXIT_MONITOR, "monitor" }, \
271 { SVM_EXIT_MWAIT, "mwait" }, \
272 { SVM_EXIT_XSETBV, "xsetbv" }, \
273 { SVM_EXIT_NPF, "npf" }
274
275/* 186/*
276 * Tracepoint for kvm guest exit: 187 * Tracepoint for kvm guest exit:
277 */ 188 */
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-23 18:49:12 -0500