diff options
author | Ingo Molnar <mingo@elte.hu> | 2008-07-10 05:43:00 -0400 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2008-07-10 05:43:00 -0400 |
commit | bac0c9103b31c3dd83ad9d731dd9834e2ba75e4f (patch) | |
tree | 702dd6a7ce06d224d594c2293af546b11ac9f51b /arch/x86 | |
parent | 6329d3021bcfa9038621e6e917d98929421d8ec8 (diff) | |
parent | 98a05ed4bd7774f533ab185fe0bf2fdc58292d7c (diff) |
Merge branch 'tracing/ftrace' into auto-ftrace-next
Diffstat (limited to 'arch/x86')
-rw-r--r-- | arch/x86/Kconfig | 2 | ||||
-rw-r--r-- | arch/x86/Kconfig.debug | 8 | ||||
-rw-r--r-- | arch/x86/kernel/Makefile | 8 | ||||
-rw-r--r-- | arch/x86/kernel/alternative.c | 22 | ||||
-rw-r--r-- | arch/x86/kernel/entry_32.S | 72 | ||||
-rw-r--r-- | arch/x86/kernel/entry_64.S | 106 | ||||
-rw-r--r-- | arch/x86/kernel/ftrace.c | 141 | ||||
-rw-r--r-- | arch/x86/kernel/i386_ksyms_32.c | 9 | ||||
-rw-r--r-- | arch/x86/kernel/machine_kexec_32.c | 4 | ||||
-rw-r--r-- | arch/x86/kernel/machine_kexec_64.c | 4 | ||||
-rw-r--r-- | arch/x86/kernel/process_32.c | 3 | ||||
-rw-r--r-- | arch/x86/kernel/process_64.c | 3 | ||||
-rw-r--r-- | arch/x86/kernel/vsyscall_64.c | 3 | ||||
-rw-r--r-- | arch/x86/kernel/x8664_ksyms_64.c | 11 | ||||
-rw-r--r-- | arch/x86/lib/Makefile | 1 | ||||
-rw-r--r-- | arch/x86/lib/thunk_32.S | 47 | ||||
-rw-r--r-- | arch/x86/lib/thunk_64.S | 19 | ||||
-rw-r--r-- | arch/x86/mm/fault.c | 56 | ||||
-rw-r--r-- | arch/x86/mm/init_32.c | 4 | ||||
-rw-r--r-- | arch/x86/mm/init_64.c | 10 | ||||
-rw-r--r-- | arch/x86/vdso/vclock_gettime.c | 15 | ||||
-rw-r--r-- | arch/x86/vdso/vgetcpu.c | 3 |
22 files changed, 524 insertions, 27 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bf07b6f50fa1..c3a4c03c0800 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig | |||
@@ -23,6 +23,8 @@ config X86 | |||
23 | select HAVE_OPROFILE | 23 | select HAVE_OPROFILE |
24 | select HAVE_KPROBES | 24 | select HAVE_KPROBES |
25 | select HAVE_KRETPROBES | 25 | select HAVE_KRETPROBES |
26 | select HAVE_DYNAMIC_FTRACE | ||
27 | select HAVE_FTRACE | ||
26 | select HAVE_KVM if ((X86_32 && !X86_VOYAGER && !X86_VISWS && !X86_NUMAQ) || X86_64) | 28 | select HAVE_KVM if ((X86_32 && !X86_VOYAGER && !X86_VISWS && !X86_NUMAQ) || X86_64) |
27 | select HAVE_ARCH_KGDB if !X86_VOYAGER | 29 | select HAVE_ARCH_KGDB if !X86_VOYAGER |
28 | 30 | ||
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug index 18363374d51a..f395fd537c5c 100644 --- a/arch/x86/Kconfig.debug +++ b/arch/x86/Kconfig.debug | |||
@@ -172,6 +172,14 @@ config IOMMU_LEAK | |||
172 | Add a simple leak tracer to the IOMMU code. This is useful when you | 172 | Add a simple leak tracer to the IOMMU code. This is useful when you |
173 | are debugging a buggy device driver that leaks IOMMU mappings. | 173 | are debugging a buggy device driver that leaks IOMMU mappings. |
174 | 174 | ||
175 | config PAGE_FAULT_HANDLERS | ||
176 | bool "Custom page fault handlers" | ||
177 | depends on DEBUG_KERNEL | ||
178 | help | ||
179 | Allow the use of custom page fault handlers. A kernel module may | ||
180 | register a function that is called on every page fault. Custom | ||
181 | handlers are used by some debugging and reverse engineering tools. | ||
182 | |||
175 | # | 183 | # |
176 | # IO delay types: | 184 | # IO delay types: |
177 | # | 185 | # |
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 77807d4769c9..5ff67208d4ae 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile | |||
@@ -6,6 +6,13 @@ extra-y := head_$(BITS).o head$(BITS).o init_task.o vmlinux.lds | |||
6 | 6 | ||
7 | CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE) | 7 | CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE) |
8 | 8 | ||
9 | ifdef CONFIG_FTRACE | ||
10 | # Do not profile debug utilities | ||
11 | CFLAGS_REMOVE_tsc_64.o = -pg | ||
12 | CFLAGS_REMOVE_tsc_32.o = -pg | ||
13 | CFLAGS_REMOVE_rtc.o = -pg | ||
14 | endif | ||
15 | |||
9 | # | 16 | # |
10 | # vsyscalls (which work on the user stack) should have | 17 | # vsyscalls (which work on the user stack) should have |
11 | # no stack-protector checks: | 18 | # no stack-protector checks: |
@@ -56,6 +63,7 @@ obj-$(CONFIG_X86_MPPARSE) += mpparse.o | |||
56 | obj-$(CONFIG_X86_LOCAL_APIC) += apic_$(BITS).o nmi_$(BITS).o | 63 | obj-$(CONFIG_X86_LOCAL_APIC) += apic_$(BITS).o nmi_$(BITS).o |
57 | obj-$(CONFIG_X86_IO_APIC) += io_apic_$(BITS).o | 64 | obj-$(CONFIG_X86_IO_APIC) += io_apic_$(BITS).o |
58 | obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups_32.o | 65 | obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups_32.o |
66 | obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o | ||
59 | obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o | 67 | obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o |
60 | obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o | 68 | obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o |
61 | obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o | 69 | obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o |
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 65c7857a90dd..2763cb37b553 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c | |||
@@ -1,6 +1,6 @@ | |||
1 | #include <linux/module.h> | 1 | #include <linux/module.h> |
2 | #include <linux/sched.h> | 2 | #include <linux/sched.h> |
3 | #include <linux/spinlock.h> | 3 | #include <linux/mutex.h> |
4 | #include <linux/list.h> | 4 | #include <linux/list.h> |
5 | #include <linux/kprobes.h> | 5 | #include <linux/kprobes.h> |
6 | #include <linux/mm.h> | 6 | #include <linux/mm.h> |
@@ -143,7 +143,7 @@ static const unsigned char *const p6_nops[ASM_NOP_MAX+1] = { | |||
143 | #ifdef CONFIG_X86_64 | 143 | #ifdef CONFIG_X86_64 |
144 | 144 | ||
145 | extern char __vsyscall_0; | 145 | extern char __vsyscall_0; |
146 | static inline const unsigned char*const * find_nop_table(void) | 146 | const unsigned char *const *find_nop_table(void) |
147 | { | 147 | { |
148 | return boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || | 148 | return boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || |
149 | boot_cpu_data.x86 < 6 ? k8_nops : p6_nops; | 149 | boot_cpu_data.x86 < 6 ? k8_nops : p6_nops; |
@@ -162,7 +162,7 @@ static const struct nop { | |||
162 | { -1, NULL } | 162 | { -1, NULL } |
163 | }; | 163 | }; |
164 | 164 | ||
165 | static const unsigned char*const * find_nop_table(void) | 165 | const unsigned char *const *find_nop_table(void) |
166 | { | 166 | { |
167 | const unsigned char *const *noptable = intel_nops; | 167 | const unsigned char *const *noptable = intel_nops; |
168 | int i; | 168 | int i; |
@@ -279,7 +279,7 @@ struct smp_alt_module { | |||
279 | struct list_head next; | 279 | struct list_head next; |
280 | }; | 280 | }; |
281 | static LIST_HEAD(smp_alt_modules); | 281 | static LIST_HEAD(smp_alt_modules); |
282 | static DEFINE_SPINLOCK(smp_alt); | 282 | static DEFINE_MUTEX(smp_alt); |
283 | static int smp_mode = 1; /* protected by smp_alt */ | 283 | static int smp_mode = 1; /* protected by smp_alt */ |
284 | 284 | ||
285 | void alternatives_smp_module_add(struct module *mod, char *name, | 285 | void alternatives_smp_module_add(struct module *mod, char *name, |
@@ -312,12 +312,12 @@ void alternatives_smp_module_add(struct module *mod, char *name, | |||
312 | __func__, smp->locks, smp->locks_end, | 312 | __func__, smp->locks, smp->locks_end, |
313 | smp->text, smp->text_end, smp->name); | 313 | smp->text, smp->text_end, smp->name); |
314 | 314 | ||
315 | spin_lock(&smp_alt); | 315 | mutex_lock(&smp_alt); |
316 | list_add_tail(&smp->next, &smp_alt_modules); | 316 | list_add_tail(&smp->next, &smp_alt_modules); |
317 | if (boot_cpu_has(X86_FEATURE_UP)) | 317 | if (boot_cpu_has(X86_FEATURE_UP)) |
318 | alternatives_smp_unlock(smp->locks, smp->locks_end, | 318 | alternatives_smp_unlock(smp->locks, smp->locks_end, |
319 | smp->text, smp->text_end); | 319 | smp->text, smp->text_end); |
320 | spin_unlock(&smp_alt); | 320 | mutex_unlock(&smp_alt); |
321 | } | 321 | } |
322 | 322 | ||
323 | void alternatives_smp_module_del(struct module *mod) | 323 | void alternatives_smp_module_del(struct module *mod) |
@@ -327,17 +327,17 @@ void alternatives_smp_module_del(struct module *mod) | |||
327 | if (smp_alt_once || noreplace_smp) | 327 | if (smp_alt_once || noreplace_smp) |
328 | return; | 328 | return; |
329 | 329 | ||
330 | spin_lock(&smp_alt); | 330 | mutex_lock(&smp_alt); |
331 | list_for_each_entry(item, &smp_alt_modules, next) { | 331 | list_for_each_entry(item, &smp_alt_modules, next) { |
332 | if (mod != item->mod) | 332 | if (mod != item->mod) |
333 | continue; | 333 | continue; |
334 | list_del(&item->next); | 334 | list_del(&item->next); |
335 | spin_unlock(&smp_alt); | 335 | mutex_unlock(&smp_alt); |
336 | DPRINTK("%s: %s\n", __func__, item->name); | 336 | DPRINTK("%s: %s\n", __func__, item->name); |
337 | kfree(item); | 337 | kfree(item); |
338 | return; | 338 | return; |
339 | } | 339 | } |
340 | spin_unlock(&smp_alt); | 340 | mutex_unlock(&smp_alt); |
341 | } | 341 | } |
342 | 342 | ||
343 | void alternatives_smp_switch(int smp) | 343 | void alternatives_smp_switch(int smp) |
@@ -359,7 +359,7 @@ void alternatives_smp_switch(int smp) | |||
359 | return; | 359 | return; |
360 | BUG_ON(!smp && (num_online_cpus() > 1)); | 360 | BUG_ON(!smp && (num_online_cpus() > 1)); |
361 | 361 | ||
362 | spin_lock(&smp_alt); | 362 | mutex_lock(&smp_alt); |
363 | 363 | ||
364 | /* | 364 | /* |
365 | * Avoid unnecessary switches because it forces JIT based VMs to | 365 | * Avoid unnecessary switches because it forces JIT based VMs to |
@@ -383,7 +383,7 @@ void alternatives_smp_switch(int smp) | |||
383 | mod->text, mod->text_end); | 383 | mod->text, mod->text_end); |
384 | } | 384 | } |
385 | smp_mode = smp; | 385 | smp_mode = smp; |
386 | spin_unlock(&smp_alt); | 386 | mutex_unlock(&smp_alt); |
387 | } | 387 | } |
388 | 388 | ||
389 | #endif | 389 | #endif |
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S index c778e4fa55a2..95e6bbe3665e 100644 --- a/arch/x86/kernel/entry_32.S +++ b/arch/x86/kernel/entry_32.S | |||
@@ -51,6 +51,7 @@ | |||
51 | #include <asm/percpu.h> | 51 | #include <asm/percpu.h> |
52 | #include <asm/dwarf2.h> | 52 | #include <asm/dwarf2.h> |
53 | #include <asm/processor-flags.h> | 53 | #include <asm/processor-flags.h> |
54 | #include <asm/ftrace.h> | ||
54 | #include "irq_vectors.h" | 55 | #include "irq_vectors.h" |
55 | 56 | ||
56 | /* | 57 | /* |
@@ -1110,6 +1111,77 @@ ENDPROC(xen_failsafe_callback) | |||
1110 | 1111 | ||
1111 | #endif /* CONFIG_XEN */ | 1112 | #endif /* CONFIG_XEN */ |
1112 | 1113 | ||
1114 | #ifdef CONFIG_FTRACE | ||
1115 | #ifdef CONFIG_DYNAMIC_FTRACE | ||
1116 | |||
1117 | ENTRY(mcount) | ||
1118 | pushl %eax | ||
1119 | pushl %ecx | ||
1120 | pushl %edx | ||
1121 | movl 0xc(%esp), %eax | ||
1122 | subl $MCOUNT_INSN_SIZE, %eax | ||
1123 | |||
1124 | .globl mcount_call | ||
1125 | mcount_call: | ||
1126 | call ftrace_stub | ||
1127 | |||
1128 | popl %edx | ||
1129 | popl %ecx | ||
1130 | popl %eax | ||
1131 | |||
1132 | ret | ||
1133 | END(mcount) | ||
1134 | |||
1135 | ENTRY(ftrace_caller) | ||
1136 | pushl %eax | ||
1137 | pushl %ecx | ||
1138 | pushl %edx | ||
1139 | movl 0xc(%esp), %eax | ||
1140 | movl 0x4(%ebp), %edx | ||
1141 | subl $MCOUNT_INSN_SIZE, %eax | ||
1142 | |||
1143 | .globl ftrace_call | ||
1144 | ftrace_call: | ||
1145 | call ftrace_stub | ||
1146 | |||
1147 | popl %edx | ||
1148 | popl %ecx | ||
1149 | popl %eax | ||
1150 | |||
1151 | .globl ftrace_stub | ||
1152 | ftrace_stub: | ||
1153 | ret | ||
1154 | END(ftrace_caller) | ||
1155 | |||
1156 | #else /* ! CONFIG_DYNAMIC_FTRACE */ | ||
1157 | |||
1158 | ENTRY(mcount) | ||
1159 | cmpl $ftrace_stub, ftrace_trace_function | ||
1160 | jnz trace | ||
1161 | .globl ftrace_stub | ||
1162 | ftrace_stub: | ||
1163 | ret | ||
1164 | |||
1165 | /* taken from glibc */ | ||
1166 | trace: | ||
1167 | pushl %eax | ||
1168 | pushl %ecx | ||
1169 | pushl %edx | ||
1170 | movl 0xc(%esp), %eax | ||
1171 | movl 0x4(%ebp), %edx | ||
1172 | subl $MCOUNT_INSN_SIZE, %eax | ||
1173 | |||
1174 | call *ftrace_trace_function | ||
1175 | |||
1176 | popl %edx | ||
1177 | popl %ecx | ||
1178 | popl %eax | ||
1179 | |||
1180 | jmp ftrace_stub | ||
1181 | END(mcount) | ||
1182 | #endif /* CONFIG_DYNAMIC_FTRACE */ | ||
1183 | #endif /* CONFIG_FTRACE */ | ||
1184 | |||
1113 | .section .rodata,"a" | 1185 | .section .rodata,"a" |
1114 | #include "syscall_table_32.S" | 1186 | #include "syscall_table_32.S" |
1115 | 1187 | ||
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 556a8df522a7..b0f7308f78a6 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S | |||
@@ -51,9 +51,115 @@ | |||
51 | #include <asm/page.h> | 51 | #include <asm/page.h> |
52 | #include <asm/irqflags.h> | 52 | #include <asm/irqflags.h> |
53 | #include <asm/paravirt.h> | 53 | #include <asm/paravirt.h> |
54 | #include <asm/ftrace.h> | ||
54 | 55 | ||
55 | .code64 | 56 | .code64 |
56 | 57 | ||
58 | #ifdef CONFIG_FTRACE | ||
59 | #ifdef CONFIG_DYNAMIC_FTRACE | ||
60 | ENTRY(mcount) | ||
61 | |||
62 | subq $0x38, %rsp | ||
63 | movq %rax, (%rsp) | ||
64 | movq %rcx, 8(%rsp) | ||
65 | movq %rdx, 16(%rsp) | ||
66 | movq %rsi, 24(%rsp) | ||
67 | movq %rdi, 32(%rsp) | ||
68 | movq %r8, 40(%rsp) | ||
69 | movq %r9, 48(%rsp) | ||
70 | |||
71 | movq 0x38(%rsp), %rdi | ||
72 | subq $MCOUNT_INSN_SIZE, %rdi | ||
73 | |||
74 | .globl mcount_call | ||
75 | mcount_call: | ||
76 | call ftrace_stub | ||
77 | |||
78 | movq 48(%rsp), %r9 | ||
79 | movq 40(%rsp), %r8 | ||
80 | movq 32(%rsp), %rdi | ||
81 | movq 24(%rsp), %rsi | ||
82 | movq 16(%rsp), %rdx | ||
83 | movq 8(%rsp), %rcx | ||
84 | movq (%rsp), %rax | ||
85 | addq $0x38, %rsp | ||
86 | |||
87 | retq | ||
88 | END(mcount) | ||
89 | |||
90 | ENTRY(ftrace_caller) | ||
91 | |||
92 | /* taken from glibc */ | ||
93 | subq $0x38, %rsp | ||
94 | movq %rax, (%rsp) | ||
95 | movq %rcx, 8(%rsp) | ||
96 | movq %rdx, 16(%rsp) | ||
97 | movq %rsi, 24(%rsp) | ||
98 | movq %rdi, 32(%rsp) | ||
99 | movq %r8, 40(%rsp) | ||
100 | movq %r9, 48(%rsp) | ||
101 | |||
102 | movq 0x38(%rsp), %rdi | ||
103 | movq 8(%rbp), %rsi | ||
104 | subq $MCOUNT_INSN_SIZE, %rdi | ||
105 | |||
106 | .globl ftrace_call | ||
107 | ftrace_call: | ||
108 | call ftrace_stub | ||
109 | |||
110 | movq 48(%rsp), %r9 | ||
111 | movq 40(%rsp), %r8 | ||
112 | movq 32(%rsp), %rdi | ||
113 | movq 24(%rsp), %rsi | ||
114 | movq 16(%rsp), %rdx | ||
115 | movq 8(%rsp), %rcx | ||
116 | movq (%rsp), %rax | ||
117 | addq $0x38, %rsp | ||
118 | |||
119 | .globl ftrace_stub | ||
120 | ftrace_stub: | ||
121 | retq | ||
122 | END(ftrace_caller) | ||
123 | |||
124 | #else /* ! CONFIG_DYNAMIC_FTRACE */ | ||
125 | ENTRY(mcount) | ||
126 | cmpq $ftrace_stub, ftrace_trace_function | ||
127 | jnz trace | ||
128 | .globl ftrace_stub | ||
129 | ftrace_stub: | ||
130 | retq | ||
131 | |||
132 | trace: | ||
133 | /* taken from glibc */ | ||
134 | subq $0x38, %rsp | ||
135 | movq %rax, (%rsp) | ||
136 | movq %rcx, 8(%rsp) | ||
137 | movq %rdx, 16(%rsp) | ||
138 | movq %rsi, 24(%rsp) | ||
139 | movq %rdi, 32(%rsp) | ||
140 | movq %r8, 40(%rsp) | ||
141 | movq %r9, 48(%rsp) | ||
142 | |||
143 | movq 0x38(%rsp), %rdi | ||
144 | movq 8(%rbp), %rsi | ||
145 | subq $MCOUNT_INSN_SIZE, %rdi | ||
146 | |||
147 | call *ftrace_trace_function | ||
148 | |||
149 | movq 48(%rsp), %r9 | ||
150 | movq 40(%rsp), %r8 | ||
151 | movq 32(%rsp), %rdi | ||
152 | movq 24(%rsp), %rsi | ||
153 | movq 16(%rsp), %rdx | ||
154 | movq 8(%rsp), %rcx | ||
155 | movq (%rsp), %rax | ||
156 | addq $0x38, %rsp | ||
157 | |||
158 | jmp ftrace_stub | ||
159 | END(mcount) | ||
160 | #endif /* CONFIG_DYNAMIC_FTRACE */ | ||
161 | #endif /* CONFIG_FTRACE */ | ||
162 | |||
57 | #ifndef CONFIG_PREEMPT | 163 | #ifndef CONFIG_PREEMPT |
58 | #define retint_kernel retint_restore_args | 164 | #define retint_kernel retint_restore_args |
59 | #endif | 165 | #endif |
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c new file mode 100644 index 000000000000..ab115cd15fdf --- /dev/null +++ b/arch/x86/kernel/ftrace.c | |||
@@ -0,0 +1,141 @@ | |||
1 | /* | ||
2 | * Code for replacing ftrace calls with jumps. | ||
3 | * | ||
4 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | ||
5 | * | ||
6 | * Thanks goes to Ingo Molnar, for suggesting the idea. | ||
7 | * Mathieu Desnoyers, for suggesting postponing the modifications. | ||
8 | * Arjan van de Ven, for keeping me straight, and explaining to me | ||
9 | * the dangers of modifying code on the run. | ||
10 | */ | ||
11 | |||
12 | #include <linux/spinlock.h> | ||
13 | #include <linux/hardirq.h> | ||
14 | #include <linux/ftrace.h> | ||
15 | #include <linux/percpu.h> | ||
16 | #include <linux/init.h> | ||
17 | #include <linux/list.h> | ||
18 | |||
19 | #include <asm/alternative.h> | ||
20 | #include <asm/ftrace.h> | ||
21 | |||
22 | |||
23 | /* Long is fine, even if it is only 4 bytes ;-) */ | ||
24 | static long *ftrace_nop; | ||
25 | |||
26 | union ftrace_code_union { | ||
27 | char code[MCOUNT_INSN_SIZE]; | ||
28 | struct { | ||
29 | char e8; | ||
30 | int offset; | ||
31 | } __attribute__((packed)); | ||
32 | }; | ||
33 | |||
34 | |||
35 | static int notrace ftrace_calc_offset(long ip, long addr) | ||
36 | { | ||
37 | return (int)(addr - ip); | ||
38 | } | ||
39 | |||
40 | notrace unsigned char *ftrace_nop_replace(void) | ||
41 | { | ||
42 | return (char *)ftrace_nop; | ||
43 | } | ||
44 | |||
45 | notrace unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) | ||
46 | { | ||
47 | static union ftrace_code_union calc; | ||
48 | |||
49 | calc.e8 = 0xe8; | ||
50 | calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr); | ||
51 | |||
52 | /* | ||
53 | * No locking needed, this must be called via kstop_machine | ||
54 | * which in essence is like running on a uniprocessor machine. | ||
55 | */ | ||
56 | return calc.code; | ||
57 | } | ||
58 | |||
59 | notrace int | ||
60 | ftrace_modify_code(unsigned long ip, unsigned char *old_code, | ||
61 | unsigned char *new_code) | ||
62 | { | ||
63 | unsigned replaced; | ||
64 | unsigned old = *(unsigned *)old_code; /* 4 bytes */ | ||
65 | unsigned new = *(unsigned *)new_code; /* 4 bytes */ | ||
66 | unsigned char newch = new_code[4]; | ||
67 | int faulted = 0; | ||
68 | |||
69 | /* | ||
70 | * Note: Due to modules and __init, code can | ||
71 | * disappear and change, we need to protect against faulting | ||
72 | * as well as code changing. | ||
73 | * | ||
74 | * No real locking needed, this code is run through | ||
75 | * kstop_machine. | ||
76 | */ | ||
77 | asm volatile ( | ||
78 | "1: lock\n" | ||
79 | " cmpxchg %3, (%2)\n" | ||
80 | " jnz 2f\n" | ||
81 | " movb %b4, 4(%2)\n" | ||
82 | "2:\n" | ||
83 | ".section .fixup, \"ax\"\n" | ||
84 | "3: movl $1, %0\n" | ||
85 | " jmp 2b\n" | ||
86 | ".previous\n" | ||
87 | _ASM_EXTABLE(1b, 3b) | ||
88 | : "=r"(faulted), "=a"(replaced) | ||
89 | : "r"(ip), "r"(new), "c"(newch), | ||
90 | "0"(faulted), "a"(old) | ||
91 | : "memory"); | ||
92 | sync_core(); | ||
93 | |||
94 | if (replaced != old && replaced != new) | ||
95 | faulted = 2; | ||
96 | |||
97 | return faulted; | ||
98 | } | ||
99 | |||
100 | notrace int ftrace_update_ftrace_func(ftrace_func_t func) | ||
101 | { | ||
102 | unsigned long ip = (unsigned long)(&ftrace_call); | ||
103 | unsigned char old[MCOUNT_INSN_SIZE], *new; | ||
104 | int ret; | ||
105 | |||
106 | memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE); | ||
107 | new = ftrace_call_replace(ip, (unsigned long)func); | ||
108 | ret = ftrace_modify_code(ip, old, new); | ||
109 | |||
110 | return ret; | ||
111 | } | ||
112 | |||
113 | notrace int ftrace_mcount_set(unsigned long *data) | ||
114 | { | ||
115 | unsigned long ip = (long)(&mcount_call); | ||
116 | unsigned long *addr = data; | ||
117 | unsigned char old[MCOUNT_INSN_SIZE], *new; | ||
118 | |||
119 | /* | ||
120 | * Replace the mcount stub with a pointer to the | ||
121 | * ip recorder function. | ||
122 | */ | ||
123 | memcpy(old, &mcount_call, MCOUNT_INSN_SIZE); | ||
124 | new = ftrace_call_replace(ip, *addr); | ||
125 | *addr = ftrace_modify_code(ip, old, new); | ||
126 | |||
127 | return 0; | ||
128 | } | ||
129 | |||
130 | int __init ftrace_dyn_arch_init(void *data) | ||
131 | { | ||
132 | const unsigned char *const *noptable = find_nop_table(); | ||
133 | |||
134 | /* This is running in kstop_machine */ | ||
135 | |||
136 | ftrace_mcount_set(data); | ||
137 | |||
138 | ftrace_nop = (unsigned long *)noptable[MCOUNT_INSN_SIZE]; | ||
139 | |||
140 | return 0; | ||
141 | } | ||
diff --git a/arch/x86/kernel/i386_ksyms_32.c b/arch/x86/kernel/i386_ksyms_32.c index deb43785e923..dd7ebee446af 100644 --- a/arch/x86/kernel/i386_ksyms_32.c +++ b/arch/x86/kernel/i386_ksyms_32.c | |||
@@ -1,7 +1,14 @@ | |||
1 | #include <linux/module.h> | 1 | #include <linux/module.h> |
2 | |||
2 | #include <asm/checksum.h> | 3 | #include <asm/checksum.h> |
3 | #include <asm/desc.h> | ||
4 | #include <asm/pgtable.h> | 4 | #include <asm/pgtable.h> |
5 | #include <asm/desc.h> | ||
6 | #include <asm/ftrace.h> | ||
7 | |||
8 | #ifdef CONFIG_FTRACE | ||
9 | /* mcount is defined in assembly */ | ||
10 | EXPORT_SYMBOL(mcount); | ||
11 | #endif | ||
5 | 12 | ||
6 | /* Networking helper routines. */ | 13 | /* Networking helper routines. */ |
7 | EXPORT_SYMBOL(csum_partial_copy_generic); | 14 | EXPORT_SYMBOL(csum_partial_copy_generic); |
diff --git a/arch/x86/kernel/machine_kexec_32.c b/arch/x86/kernel/machine_kexec_32.c index d0b234c9fc31..88923fd7a6fc 100644 --- a/arch/x86/kernel/machine_kexec_32.c +++ b/arch/x86/kernel/machine_kexec_32.c | |||
@@ -11,6 +11,8 @@ | |||
11 | #include <linux/delay.h> | 11 | #include <linux/delay.h> |
12 | #include <linux/init.h> | 12 | #include <linux/init.h> |
13 | #include <linux/numa.h> | 13 | #include <linux/numa.h> |
14 | #include <linux/ftrace.h> | ||
15 | |||
14 | #include <asm/pgtable.h> | 16 | #include <asm/pgtable.h> |
15 | #include <asm/pgalloc.h> | 17 | #include <asm/pgalloc.h> |
16 | #include <asm/tlbflush.h> | 18 | #include <asm/tlbflush.h> |
@@ -107,6 +109,8 @@ NORET_TYPE void machine_kexec(struct kimage *image) | |||
107 | unsigned long page_list[PAGES_NR]; | 109 | unsigned long page_list[PAGES_NR]; |
108 | void *control_page; | 110 | void *control_page; |
109 | 111 | ||
112 | tracer_disable(); | ||
113 | |||
110 | /* Interrupts aren't acceptable while we reboot */ | 114 | /* Interrupts aren't acceptable while we reboot */ |
111 | local_irq_disable(); | 115 | local_irq_disable(); |
112 | 116 | ||
diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c index 576a03db4511..1558fdc174f9 100644 --- a/arch/x86/kernel/machine_kexec_64.c +++ b/arch/x86/kernel/machine_kexec_64.c | |||
@@ -11,6 +11,8 @@ | |||
11 | #include <linux/string.h> | 11 | #include <linux/string.h> |
12 | #include <linux/reboot.h> | 12 | #include <linux/reboot.h> |
13 | #include <linux/numa.h> | 13 | #include <linux/numa.h> |
14 | #include <linux/ftrace.h> | ||
15 | |||
14 | #include <asm/pgtable.h> | 16 | #include <asm/pgtable.h> |
15 | #include <asm/tlbflush.h> | 17 | #include <asm/tlbflush.h> |
16 | #include <asm/mmu_context.h> | 18 | #include <asm/mmu_context.h> |
@@ -184,6 +186,8 @@ NORET_TYPE void machine_kexec(struct kimage *image) | |||
184 | unsigned long page_list[PAGES_NR]; | 186 | unsigned long page_list[PAGES_NR]; |
185 | void *control_page; | 187 | void *control_page; |
186 | 188 | ||
189 | tracer_disable(); | ||
190 | |||
187 | /* Interrupts aren't acceptable while we reboot */ | 191 | /* Interrupts aren't acceptable while we reboot */ |
188 | local_irq_disable(); | 192 | local_irq_disable(); |
189 | 193 | ||
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index e2db9ac5c61c..347a7aba8b16 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c | |||
@@ -185,7 +185,10 @@ void cpu_idle(void) | |||
185 | 185 | ||
186 | local_irq_disable(); | 186 | local_irq_disable(); |
187 | __get_cpu_var(irq_stat).idle_timestamp = jiffies; | 187 | __get_cpu_var(irq_stat).idle_timestamp = jiffies; |
188 | /* Don't trace irqs off for idle */ | ||
189 | stop_critical_timings(); | ||
188 | idle(); | 190 | idle(); |
191 | start_critical_timings(); | ||
189 | } | 192 | } |
190 | tick_nohz_restart_sched_tick(); | 193 | tick_nohz_restart_sched_tick(); |
191 | preempt_enable_no_resched(); | 194 | preempt_enable_no_resched(); |
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index c6eb5c91e5f6..ea090e6cfe39 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c | |||
@@ -165,7 +165,10 @@ void cpu_idle(void) | |||
165 | */ | 165 | */ |
166 | local_irq_disable(); | 166 | local_irq_disable(); |
167 | enter_idle(); | 167 | enter_idle(); |
168 | /* Don't trace irqs off for idle */ | ||
169 | stop_critical_timings(); | ||
168 | idle(); | 170 | idle(); |
171 | start_critical_timings(); | ||
169 | /* In many cases the interrupt that ended idle | 172 | /* In many cases the interrupt that ended idle |
170 | has already called exit_idle. But some idle | 173 | has already called exit_idle. But some idle |
171 | loops can be woken up without interrupt. */ | 174 | loops can be woken up without interrupt. */ |
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index 61efa2f7d564..4063dfa2a02d 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c | |||
@@ -42,7 +42,8 @@ | |||
42 | #include <asm/topology.h> | 42 | #include <asm/topology.h> |
43 | #include <asm/vgtod.h> | 43 | #include <asm/vgtod.h> |
44 | 44 | ||
45 | #define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr))) | 45 | #define __vsyscall(nr) \ |
46 | __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace | ||
46 | #define __syscall_clobber "r11","cx","memory" | 47 | #define __syscall_clobber "r11","cx","memory" |
47 | 48 | ||
48 | /* | 49 | /* |
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c index f6c05d0410fb..16ff4bf418d9 100644 --- a/arch/x86/kernel/x8664_ksyms_64.c +++ b/arch/x86/kernel/x8664_ksyms_64.c | |||
@@ -2,13 +2,20 @@ | |||
2 | All C exports should go in the respective C files. */ | 2 | All C exports should go in the respective C files. */ |
3 | 3 | ||
4 | #include <linux/module.h> | 4 | #include <linux/module.h> |
5 | #include <net/checksum.h> | ||
6 | #include <linux/smp.h> | 5 | #include <linux/smp.h> |
7 | 6 | ||
7 | #include <net/checksum.h> | ||
8 | |||
8 | #include <asm/processor.h> | 9 | #include <asm/processor.h> |
9 | #include <asm/uaccess.h> | ||
10 | #include <asm/pgtable.h> | 10 | #include <asm/pgtable.h> |
11 | #include <asm/uaccess.h> | ||
11 | #include <asm/desc.h> | 12 | #include <asm/desc.h> |
13 | #include <asm/ftrace.h> | ||
14 | |||
15 | #ifdef CONFIG_FTRACE | ||
16 | /* mcount is defined in assembly */ | ||
17 | EXPORT_SYMBOL(mcount); | ||
18 | #endif | ||
12 | 19 | ||
13 | EXPORT_SYMBOL(kernel_thread); | 20 | EXPORT_SYMBOL(kernel_thread); |
14 | 21 | ||
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 76f60f52a885..84aa2883fe15 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile | |||
@@ -5,6 +5,7 @@ | |||
5 | obj-$(CONFIG_SMP) := msr-on-cpu.o | 5 | obj-$(CONFIG_SMP) := msr-on-cpu.o |
6 | 6 | ||
7 | lib-y := delay_$(BITS).o | 7 | lib-y := delay_$(BITS).o |
8 | lib-y += thunk_$(BITS).o | ||
8 | lib-y += usercopy_$(BITS).o getuser_$(BITS).o putuser_$(BITS).o | 9 | lib-y += usercopy_$(BITS).o getuser_$(BITS).o putuser_$(BITS).o |
9 | lib-y += memcpy_$(BITS).o | 10 | lib-y += memcpy_$(BITS).o |
10 | 11 | ||
diff --git a/arch/x86/lib/thunk_32.S b/arch/x86/lib/thunk_32.S new file mode 100644 index 000000000000..650b11e00ecc --- /dev/null +++ b/arch/x86/lib/thunk_32.S | |||
@@ -0,0 +1,47 @@ | |||
1 | /* | ||
2 | * Trampoline to trace irqs off. (otherwise CALLER_ADDR1 might crash) | ||
3 | * Copyright 2008 by Steven Rostedt, Red Hat, Inc | ||
4 | * (inspired by Andi Kleen's thunk_64.S) | ||
5 | * Subject to the GNU public license, v.2. No warranty of any kind. | ||
6 | */ | ||
7 | |||
8 | #include <linux/linkage.h> | ||
9 | |||
10 | #define ARCH_TRACE_IRQS_ON \ | ||
11 | pushl %eax; \ | ||
12 | pushl %ecx; \ | ||
13 | pushl %edx; \ | ||
14 | call trace_hardirqs_on; \ | ||
15 | popl %edx; \ | ||
16 | popl %ecx; \ | ||
17 | popl %eax; | ||
18 | |||
19 | #define ARCH_TRACE_IRQS_OFF \ | ||
20 | pushl %eax; \ | ||
21 | pushl %ecx; \ | ||
22 | pushl %edx; \ | ||
23 | call trace_hardirqs_off; \ | ||
24 | popl %edx; \ | ||
25 | popl %ecx; \ | ||
26 | popl %eax; | ||
27 | |||
28 | #ifdef CONFIG_TRACE_IRQFLAGS | ||
29 | /* put return address in eax (arg1) */ | ||
30 | .macro thunk_ra name,func | ||
31 | .globl \name | ||
32 | \name: | ||
33 | pushl %eax | ||
34 | pushl %ecx | ||
35 | pushl %edx | ||
36 | /* Place EIP in the arg1 */ | ||
37 | movl 3*4(%esp), %eax | ||
38 | call \func | ||
39 | popl %edx | ||
40 | popl %ecx | ||
41 | popl %eax | ||
42 | ret | ||
43 | .endm | ||
44 | |||
45 | thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller | ||
46 | thunk_ra trace_hardirqs_off_thunk,trace_hardirqs_off_caller | ||
47 | #endif | ||
diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S index e009251d4e9f..bf9a7d5a5428 100644 --- a/arch/x86/lib/thunk_64.S +++ b/arch/x86/lib/thunk_64.S | |||
@@ -2,6 +2,7 @@ | |||
2 | * Save registers before calling assembly functions. This avoids | 2 | * Save registers before calling assembly functions. This avoids |
3 | * disturbance of register allocation in some inline assembly constructs. | 3 | * disturbance of register allocation in some inline assembly constructs. |
4 | * Copyright 2001,2002 by Andi Kleen, SuSE Labs. | 4 | * Copyright 2001,2002 by Andi Kleen, SuSE Labs. |
5 | * Added trace_hardirqs callers - Copyright 2007 Steven Rostedt, Red Hat, Inc. | ||
5 | * Subject to the GNU public license, v.2. No warranty of any kind. | 6 | * Subject to the GNU public license, v.2. No warranty of any kind. |
6 | */ | 7 | */ |
7 | 8 | ||
@@ -42,8 +43,22 @@ | |||
42 | #endif | 43 | #endif |
43 | 44 | ||
44 | #ifdef CONFIG_TRACE_IRQFLAGS | 45 | #ifdef CONFIG_TRACE_IRQFLAGS |
45 | thunk trace_hardirqs_on_thunk,trace_hardirqs_on | 46 | /* put return address in rdi (arg1) */ |
46 | thunk trace_hardirqs_off_thunk,trace_hardirqs_off | 47 | .macro thunk_ra name,func |
48 | .globl \name | ||
49 | \name: | ||
50 | CFI_STARTPROC | ||
51 | SAVE_ARGS | ||
52 | /* SAVE_ARGS pushs 9 elements */ | ||
53 | /* the next element would be the rip */ | ||
54 | movq 9*8(%rsp), %rdi | ||
55 | call \func | ||
56 | jmp restore | ||
57 | CFI_ENDPROC | ||
58 | .endm | ||
59 | |||
60 | thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller | ||
61 | thunk_ra trace_hardirqs_off_thunk,trace_hardirqs_off_caller | ||
47 | #endif | 62 | #endif |
48 | 63 | ||
49 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 64 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 8bcb6f40ccb6..42394b353c6a 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c | |||
@@ -49,6 +49,60 @@ | |||
49 | #define PF_RSVD (1<<3) | 49 | #define PF_RSVD (1<<3) |
50 | #define PF_INSTR (1<<4) | 50 | #define PF_INSTR (1<<4) |
51 | 51 | ||
52 | #ifdef CONFIG_PAGE_FAULT_HANDLERS | ||
53 | static HLIST_HEAD(pf_handlers); /* protected by RCU */ | ||
54 | static DEFINE_SPINLOCK(pf_handlers_writer); | ||
55 | |||
56 | void register_page_fault_handler(struct pf_handler *new_pfh) | ||
57 | { | ||
58 | unsigned long flags; | ||
59 | spin_lock_irqsave(&pf_handlers_writer, flags); | ||
60 | hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers); | ||
61 | spin_unlock_irqrestore(&pf_handlers_writer, flags); | ||
62 | } | ||
63 | EXPORT_SYMBOL_GPL(register_page_fault_handler); | ||
64 | |||
65 | /** | ||
66 | * unregister_page_fault_handler: | ||
67 | * The caller must ensure @old_pfh is not in use anymore before freeing it. | ||
68 | * This function does not guarantee it. The list of handlers is protected by | ||
69 | * RCU, so you can do this by e.g. calling synchronize_rcu(). | ||
70 | */ | ||
71 | void unregister_page_fault_handler(struct pf_handler *old_pfh) | ||
72 | { | ||
73 | unsigned long flags; | ||
74 | spin_lock_irqsave(&pf_handlers_writer, flags); | ||
75 | hlist_del_rcu(&old_pfh->hlist); | ||
76 | spin_unlock_irqrestore(&pf_handlers_writer, flags); | ||
77 | } | ||
78 | EXPORT_SYMBOL_GPL(unregister_page_fault_handler); | ||
79 | #endif | ||
80 | |||
81 | /* returns non-zero if do_page_fault() should return */ | ||
82 | static int handle_custom_pf(struct pt_regs *regs, unsigned long error_code, | ||
83 | unsigned long address) | ||
84 | { | ||
85 | #ifdef CONFIG_PAGE_FAULT_HANDLERS | ||
86 | int ret = 0; | ||
87 | struct pf_handler *cur; | ||
88 | struct hlist_node *ncur; | ||
89 | |||
90 | if (hlist_empty(&pf_handlers)) | ||
91 | return 0; | ||
92 | |||
93 | rcu_read_lock(); | ||
94 | hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) { | ||
95 | ret = cur->handler(regs, error_code, address); | ||
96 | if (ret) | ||
97 | break; | ||
98 | } | ||
99 | rcu_read_unlock(); | ||
100 | return ret; | ||
101 | #else | ||
102 | return 0; | ||
103 | #endif | ||
104 | } | ||
105 | |||
52 | static inline int notify_page_fault(struct pt_regs *regs) | 106 | static inline int notify_page_fault(struct pt_regs *regs) |
53 | { | 107 | { |
54 | #ifdef CONFIG_KPROBES | 108 | #ifdef CONFIG_KPROBES |
@@ -606,6 +660,8 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) | |||
606 | 660 | ||
607 | if (notify_page_fault(regs)) | 661 | if (notify_page_fault(regs)) |
608 | return; | 662 | return; |
663 | if (handle_custom_pf(regs, error_code, address)) | ||
664 | return; | ||
609 | 665 | ||
610 | /* | 666 | /* |
611 | * We fault-in kernel-space virtual memory on-demand. The | 667 | * We fault-in kernel-space virtual memory on-demand. The |
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index ec30d10154b6..f96eca21ad8f 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c | |||
@@ -710,6 +710,8 @@ void mark_rodata_ro(void) | |||
710 | unsigned long start = PFN_ALIGN(_text); | 710 | unsigned long start = PFN_ALIGN(_text); |
711 | unsigned long size = PFN_ALIGN(_etext) - start; | 711 | unsigned long size = PFN_ALIGN(_etext) - start; |
712 | 712 | ||
713 | #ifndef CONFIG_DYNAMIC_FTRACE | ||
714 | /* Dynamic tracing modifies the kernel text section */ | ||
713 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); | 715 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
714 | printk(KERN_INFO "Write protecting the kernel text: %luk\n", | 716 | printk(KERN_INFO "Write protecting the kernel text: %luk\n", |
715 | size >> 10); | 717 | size >> 10); |
@@ -722,6 +724,8 @@ void mark_rodata_ro(void) | |||
722 | printk(KERN_INFO "Testing CPA: write protecting again\n"); | 724 | printk(KERN_INFO "Testing CPA: write protecting again\n"); |
723 | set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT); | 725 | set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT); |
724 | #endif | 726 | #endif |
727 | #endif /* CONFIG_DYNAMIC_FTRACE */ | ||
728 | |||
725 | start += size; | 729 | start += size; |
726 | size = (unsigned long)__end_rodata - start; | 730 | size = (unsigned long)__end_rodata - start; |
727 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); | 731 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 819dad973b13..17c0a6138a53 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c | |||
@@ -767,6 +767,13 @@ EXPORT_SYMBOL_GPL(rodata_test_data); | |||
767 | void mark_rodata_ro(void) | 767 | void mark_rodata_ro(void) |
768 | { | 768 | { |
769 | unsigned long start = PFN_ALIGN(_stext), end = PFN_ALIGN(__end_rodata); | 769 | unsigned long start = PFN_ALIGN(_stext), end = PFN_ALIGN(__end_rodata); |
770 | unsigned long rodata_start = | ||
771 | ((unsigned long)__start_rodata + PAGE_SIZE - 1) & PAGE_MASK; | ||
772 | |||
773 | #ifdef CONFIG_DYNAMIC_FTRACE | ||
774 | /* Dynamic tracing modifies the kernel text section */ | ||
775 | start = rodata_start; | ||
776 | #endif | ||
770 | 777 | ||
771 | printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", | 778 | printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", |
772 | (end - start) >> 10); | 779 | (end - start) >> 10); |
@@ -776,8 +783,7 @@ void mark_rodata_ro(void) | |||
776 | * The rodata section (but not the kernel text!) should also be | 783 | * The rodata section (but not the kernel text!) should also be |
777 | * not-executable. | 784 | * not-executable. |
778 | */ | 785 | */ |
779 | start = ((unsigned long)__start_rodata + PAGE_SIZE - 1) & PAGE_MASK; | 786 | set_memory_nx(rodata_start, (end - rodata_start) >> PAGE_SHIFT); |
780 | set_memory_nx(start, (end - start) >> PAGE_SHIFT); | ||
781 | 787 | ||
782 | rodata_test(); | 788 | rodata_test(); |
783 | 789 | ||
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c index efa2ba7c6005..1ef0f90813d6 100644 --- a/arch/x86/vdso/vclock_gettime.c +++ b/arch/x86/vdso/vclock_gettime.c | |||
@@ -23,7 +23,7 @@ | |||
23 | 23 | ||
24 | #define gtod vdso_vsyscall_gtod_data | 24 | #define gtod vdso_vsyscall_gtod_data |
25 | 25 | ||
26 | static long vdso_fallback_gettime(long clock, struct timespec *ts) | 26 | notrace static long vdso_fallback_gettime(long clock, struct timespec *ts) |
27 | { | 27 | { |
28 | long ret; | 28 | long ret; |
29 | asm("syscall" : "=a" (ret) : | 29 | asm("syscall" : "=a" (ret) : |
@@ -31,7 +31,7 @@ static long vdso_fallback_gettime(long clock, struct timespec *ts) | |||
31 | return ret; | 31 | return ret; |
32 | } | 32 | } |
33 | 33 | ||
34 | static inline long vgetns(void) | 34 | notrace static inline long vgetns(void) |
35 | { | 35 | { |
36 | long v; | 36 | long v; |
37 | cycles_t (*vread)(void); | 37 | cycles_t (*vread)(void); |
@@ -40,7 +40,7 @@ static inline long vgetns(void) | |||
40 | return (v * gtod->clock.mult) >> gtod->clock.shift; | 40 | return (v * gtod->clock.mult) >> gtod->clock.shift; |
41 | } | 41 | } |
42 | 42 | ||
43 | static noinline int do_realtime(struct timespec *ts) | 43 | notrace static noinline int do_realtime(struct timespec *ts) |
44 | { | 44 | { |
45 | unsigned long seq, ns; | 45 | unsigned long seq, ns; |
46 | do { | 46 | do { |
@@ -54,7 +54,8 @@ static noinline int do_realtime(struct timespec *ts) | |||
54 | } | 54 | } |
55 | 55 | ||
56 | /* Copy of the version in kernel/time.c which we cannot directly access */ | 56 | /* Copy of the version in kernel/time.c which we cannot directly access */ |
57 | static void vset_normalized_timespec(struct timespec *ts, long sec, long nsec) | 57 | notrace static void |
58 | vset_normalized_timespec(struct timespec *ts, long sec, long nsec) | ||
58 | { | 59 | { |
59 | while (nsec >= NSEC_PER_SEC) { | 60 | while (nsec >= NSEC_PER_SEC) { |
60 | nsec -= NSEC_PER_SEC; | 61 | nsec -= NSEC_PER_SEC; |
@@ -68,7 +69,7 @@ static void vset_normalized_timespec(struct timespec *ts, long sec, long nsec) | |||
68 | ts->tv_nsec = nsec; | 69 | ts->tv_nsec = nsec; |
69 | } | 70 | } |
70 | 71 | ||
71 | static noinline int do_monotonic(struct timespec *ts) | 72 | notrace static noinline int do_monotonic(struct timespec *ts) |
72 | { | 73 | { |
73 | unsigned long seq, ns, secs; | 74 | unsigned long seq, ns, secs; |
74 | do { | 75 | do { |
@@ -82,7 +83,7 @@ static noinline int do_monotonic(struct timespec *ts) | |||
82 | return 0; | 83 | return 0; |
83 | } | 84 | } |
84 | 85 | ||
85 | int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) | 86 | notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) |
86 | { | 87 | { |
87 | if (likely(gtod->sysctl_enabled && gtod->clock.vread)) | 88 | if (likely(gtod->sysctl_enabled && gtod->clock.vread)) |
88 | switch (clock) { | 89 | switch (clock) { |
@@ -96,7 +97,7 @@ int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) | |||
96 | int clock_gettime(clockid_t, struct timespec *) | 97 | int clock_gettime(clockid_t, struct timespec *) |
97 | __attribute__((weak, alias("__vdso_clock_gettime"))); | 98 | __attribute__((weak, alias("__vdso_clock_gettime"))); |
98 | 99 | ||
99 | int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) | 100 | notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) |
100 | { | 101 | { |
101 | long ret; | 102 | long ret; |
102 | if (likely(gtod->sysctl_enabled && gtod->clock.vread)) { | 103 | if (likely(gtod->sysctl_enabled && gtod->clock.vread)) { |
diff --git a/arch/x86/vdso/vgetcpu.c b/arch/x86/vdso/vgetcpu.c index c8097f17f8a9..9fbc6b20026b 100644 --- a/arch/x86/vdso/vgetcpu.c +++ b/arch/x86/vdso/vgetcpu.c | |||
@@ -13,7 +13,8 @@ | |||
13 | #include <asm/vgtod.h> | 13 | #include <asm/vgtod.h> |
14 | #include "vextern.h" | 14 | #include "vextern.h" |
15 | 15 | ||
16 | long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) | 16 | notrace long |
17 | __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) | ||
17 | { | 18 | { |
18 | unsigned int p; | 19 | unsigned int p; |
19 | 20 | ||