diff options
Diffstat (limited to 'arch')
37 files changed, 627 insertions, 132 deletions
diff --git a/arch/Kconfig b/arch/Kconfig index 830c16a2b801..dc81b34c5d82 100644 --- a/arch/Kconfig +++ b/arch/Kconfig | |||
@@ -6,6 +6,7 @@ config OPROFILE | |||
6 | tristate "OProfile system profiling (EXPERIMENTAL)" | 6 | tristate "OProfile system profiling (EXPERIMENTAL)" |
7 | depends on PROFILING | 7 | depends on PROFILING |
8 | depends on HAVE_OPROFILE | 8 | depends on HAVE_OPROFILE |
9 | depends on TRACING_SUPPORT | ||
9 | select TRACING | 10 | select TRACING |
10 | select RING_BUFFER | 11 | select RING_BUFFER |
11 | help | 12 | help |
diff --git a/arch/alpha/include/asm/ftrace.h b/arch/alpha/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/alpha/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||
diff --git a/arch/alpha/include/asm/hardirq.h b/arch/alpha/include/asm/hardirq.h index d953e234daa8..88971460fa6c 100644 --- a/arch/alpha/include/asm/hardirq.h +++ b/arch/alpha/include/asm/hardirq.h | |||
@@ -14,17 +14,4 @@ typedef struct { | |||
14 | 14 | ||
15 | void ack_bad_irq(unsigned int irq); | 15 | void ack_bad_irq(unsigned int irq); |
16 | 16 | ||
17 | #define HARDIRQ_BITS 12 | ||
18 | |||
19 | /* | ||
20 | * The hardirq mask has to be large enough to have | ||
21 | * space for potentially nestable IRQ sources in the system | ||
22 | * to nest on a single CPU. On Alpha, interrupts are masked at the CPU | ||
23 | * by IPL as well as at the system level. We only have 8 IPLs (UNIX PALcode) | ||
24 | * so we really only have 8 nestable IRQs, but allow some overhead | ||
25 | */ | ||
26 | #if (1 << HARDIRQ_BITS) < 16 | ||
27 | #error HARDIRQ_BITS is too low! | ||
28 | #endif | ||
29 | |||
30 | #endif /* _ALPHA_HARDIRQ_H */ | 17 | #endif /* _ALPHA_HARDIRQ_H */ |
diff --git a/arch/avr32/include/asm/ftrace.h b/arch/avr32/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/avr32/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||
diff --git a/arch/avr32/include/asm/hardirq.h b/arch/avr32/include/asm/hardirq.h index 267354356f60..015bc75ea798 100644 --- a/arch/avr32/include/asm/hardirq.h +++ b/arch/avr32/include/asm/hardirq.h | |||
@@ -20,15 +20,4 @@ void ack_bad_irq(unsigned int irq); | |||
20 | 20 | ||
21 | #endif /* __ASSEMBLY__ */ | 21 | #endif /* __ASSEMBLY__ */ |
22 | 22 | ||
23 | #define HARDIRQ_BITS 12 | ||
24 | |||
25 | /* | ||
26 | * The hardirq mask has to be large enough to have | ||
27 | * space for potentially all IRQ sources in the system | ||
28 | * nesting on a single CPU: | ||
29 | */ | ||
30 | #if (1 << HARDIRQ_BITS) < NR_IRQS | ||
31 | # error HARDIRQ_BITS is too low! | ||
32 | #endif | ||
33 | |||
34 | #endif /* __ASM_AVR32_HARDIRQ_H */ | 23 | #endif /* __ASM_AVR32_HARDIRQ_H */ |
diff --git a/arch/blackfin/include/asm/ftrace.h b/arch/blackfin/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/blackfin/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||
diff --git a/arch/cris/include/asm/ftrace.h b/arch/cris/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/cris/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||
diff --git a/arch/h8300/include/asm/ftrace.h b/arch/h8300/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/h8300/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 153e727a6e8e..294a3b13ecac 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig | |||
@@ -22,6 +22,9 @@ config IA64 | |||
22 | select HAVE_OPROFILE | 22 | select HAVE_OPROFILE |
23 | select HAVE_KPROBES | 23 | select HAVE_KPROBES |
24 | select HAVE_KRETPROBES | 24 | select HAVE_KRETPROBES |
25 | select HAVE_FTRACE_MCOUNT_RECORD | ||
26 | select HAVE_DYNAMIC_FTRACE if (!ITANIUM) | ||
27 | select HAVE_FUNCTION_TRACER | ||
25 | select HAVE_DMA_ATTRS | 28 | select HAVE_DMA_ATTRS |
26 | select HAVE_KVM | 29 | select HAVE_KVM |
27 | select HAVE_ARCH_TRACEHOOK | 30 | select HAVE_ARCH_TRACEHOOK |
diff --git a/arch/ia64/include/asm/ftrace.h b/arch/ia64/include/asm/ftrace.h new file mode 100644 index 000000000000..d20db3c2a656 --- /dev/null +++ b/arch/ia64/include/asm/ftrace.h | |||
@@ -0,0 +1,28 @@ | |||
1 | #ifndef _ASM_IA64_FTRACE_H | ||
2 | #define _ASM_IA64_FTRACE_H | ||
3 | |||
4 | #ifdef CONFIG_FUNCTION_TRACER | ||
5 | #define MCOUNT_INSN_SIZE 32 /* sizeof mcount call */ | ||
6 | |||
7 | #ifndef __ASSEMBLY__ | ||
8 | extern void _mcount(unsigned long pfs, unsigned long r1, unsigned long b0, unsigned long r0); | ||
9 | #define mcount _mcount | ||
10 | |||
11 | #include <asm/kprobes.h> | ||
12 | /* In IA64, MCOUNT_ADDR is set in link time, so it's not a constant at compile time */ | ||
13 | #define MCOUNT_ADDR (((struct fnptr *)mcount)->ip) | ||
14 | #define FTRACE_ADDR (((struct fnptr *)ftrace_caller)->ip) | ||
15 | |||
16 | static inline unsigned long ftrace_call_adjust(unsigned long addr) | ||
17 | { | ||
18 | /* second bundle, insn 2 */ | ||
19 | return addr - 0x12; | ||
20 | } | ||
21 | |||
22 | struct dyn_arch_ftrace { | ||
23 | }; | ||
24 | #endif | ||
25 | |||
26 | #endif /* CONFIG_FUNCTION_TRACER */ | ||
27 | |||
28 | #endif /* _ASM_IA64_FTRACE_H */ | ||
diff --git a/arch/ia64/include/asm/hardirq.h b/arch/ia64/include/asm/hardirq.h index 140e495b8e0e..d514cd9edb49 100644 --- a/arch/ia64/include/asm/hardirq.h +++ b/arch/ia64/include/asm/hardirq.h | |||
@@ -20,16 +20,6 @@ | |||
20 | 20 | ||
21 | #define local_softirq_pending() (local_cpu_data->softirq_pending) | 21 | #define local_softirq_pending() (local_cpu_data->softirq_pending) |
22 | 22 | ||
23 | #define HARDIRQ_BITS 14 | ||
24 | |||
25 | /* | ||
26 | * The hardirq mask has to be large enough to have space for potentially all IRQ sources | ||
27 | * in the system nesting on a single CPU: | ||
28 | */ | ||
29 | #if (1 << HARDIRQ_BITS) < NR_IRQS | ||
30 | # error HARDIRQ_BITS is too low! | ||
31 | #endif | ||
32 | |||
33 | extern void __iomem *ipi_base_addr; | 23 | extern void __iomem *ipi_base_addr; |
34 | 24 | ||
35 | void ack_bad_irq(unsigned int irq); | 25 | void ack_bad_irq(unsigned int irq); |
diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile index f2778f2c4fd9..dc62df021673 100644 --- a/arch/ia64/kernel/Makefile +++ b/arch/ia64/kernel/Makefile | |||
@@ -2,6 +2,10 @@ | |||
2 | # Makefile for the linux kernel. | 2 | # Makefile for the linux kernel. |
3 | # | 3 | # |
4 | 4 | ||
5 | ifdef CONFIG_DYNAMIC_FTRACE | ||
6 | CFLAGS_REMOVE_ftrace.o = -pg | ||
7 | endif | ||
8 | |||
5 | extra-y := head.o init_task.o vmlinux.lds | 9 | extra-y := head.o init_task.o vmlinux.lds |
6 | 10 | ||
7 | obj-y := acpi.o entry.o efi.o efi_stub.o gate-data.o fsys.o ia64_ksyms.o irq.o irq_ia64.o \ | 11 | obj-y := acpi.o entry.o efi.o efi_stub.o gate-data.o fsys.o ia64_ksyms.o irq.o irq_ia64.o \ |
@@ -28,6 +32,7 @@ obj-$(CONFIG_IA64_CYCLONE) += cyclone.o | |||
28 | obj-$(CONFIG_CPU_FREQ) += cpufreq/ | 32 | obj-$(CONFIG_CPU_FREQ) += cpufreq/ |
29 | obj-$(CONFIG_IA64_MCA_RECOVERY) += mca_recovery.o | 33 | obj-$(CONFIG_IA64_MCA_RECOVERY) += mca_recovery.o |
30 | obj-$(CONFIG_KPROBES) += kprobes.o jprobes.o | 34 | obj-$(CONFIG_KPROBES) += kprobes.o jprobes.o |
35 | obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o | ||
31 | obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o crash.o | 36 | obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o crash.o |
32 | obj-$(CONFIG_CRASH_DUMP) += crash_dump.o | 37 | obj-$(CONFIG_CRASH_DUMP) += crash_dump.o |
33 | obj-$(CONFIG_IA64_UNCACHED_ALLOCATOR) += uncached.o | 38 | obj-$(CONFIG_IA64_UNCACHED_ALLOCATOR) += uncached.o |
diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S index e5341e2c1175..7e3382b06d56 100644 --- a/arch/ia64/kernel/entry.S +++ b/arch/ia64/kernel/entry.S | |||
@@ -47,6 +47,7 @@ | |||
47 | #include <asm/processor.h> | 47 | #include <asm/processor.h> |
48 | #include <asm/thread_info.h> | 48 | #include <asm/thread_info.h> |
49 | #include <asm/unistd.h> | 49 | #include <asm/unistd.h> |
50 | #include <asm/ftrace.h> | ||
50 | 51 | ||
51 | #include "minstate.h" | 52 | #include "minstate.h" |
52 | 53 | ||
@@ -1404,6 +1405,105 @@ GLOBAL_ENTRY(unw_init_running) | |||
1404 | br.ret.sptk.many rp | 1405 | br.ret.sptk.many rp |
1405 | END(unw_init_running) | 1406 | END(unw_init_running) |
1406 | 1407 | ||
1408 | #ifdef CONFIG_FUNCTION_TRACER | ||
1409 | #ifdef CONFIG_DYNAMIC_FTRACE | ||
1410 | GLOBAL_ENTRY(_mcount) | ||
1411 | br ftrace_stub | ||
1412 | END(_mcount) | ||
1413 | |||
1414 | .here: | ||
1415 | br.ret.sptk.many b0 | ||
1416 | |||
1417 | GLOBAL_ENTRY(ftrace_caller) | ||
1418 | alloc out0 = ar.pfs, 8, 0, 4, 0 | ||
1419 | mov out3 = r0 | ||
1420 | ;; | ||
1421 | mov out2 = b0 | ||
1422 | add r3 = 0x20, r3 | ||
1423 | mov out1 = r1; | ||
1424 | br.call.sptk.many b0 = ftrace_patch_gp | ||
1425 | //this might be called from module, so we must patch gp | ||
1426 | ftrace_patch_gp: | ||
1427 | movl gp=__gp | ||
1428 | mov b0 = r3 | ||
1429 | ;; | ||
1430 | .global ftrace_call; | ||
1431 | ftrace_call: | ||
1432 | { | ||
1433 | .mlx | ||
1434 | nop.m 0x0 | ||
1435 | movl r3 = .here;; | ||
1436 | } | ||
1437 | alloc loc0 = ar.pfs, 4, 4, 2, 0 | ||
1438 | ;; | ||
1439 | mov loc1 = b0 | ||
1440 | mov out0 = b0 | ||
1441 | mov loc2 = r8 | ||
1442 | mov loc3 = r15 | ||
1443 | ;; | ||
1444 | adds out0 = -MCOUNT_INSN_SIZE, out0 | ||
1445 | mov out1 = in2 | ||
1446 | mov b6 = r3 | ||
1447 | |||
1448 | br.call.sptk.many b0 = b6 | ||
1449 | ;; | ||
1450 | mov ar.pfs = loc0 | ||
1451 | mov b0 = loc1 | ||
1452 | mov r8 = loc2 | ||
1453 | mov r15 = loc3 | ||
1454 | br ftrace_stub | ||
1455 | ;; | ||
1456 | END(ftrace_caller) | ||
1457 | |||
1458 | #else | ||
1459 | GLOBAL_ENTRY(_mcount) | ||
1460 | movl r2 = ftrace_stub | ||
1461 | movl r3 = ftrace_trace_function;; | ||
1462 | ld8 r3 = [r3];; | ||
1463 | ld8 r3 = [r3];; | ||
1464 | cmp.eq p7,p0 = r2, r3 | ||
1465 | (p7) br.sptk.many ftrace_stub | ||
1466 | ;; | ||
1467 | |||
1468 | alloc loc0 = ar.pfs, 4, 4, 2, 0 | ||
1469 | ;; | ||
1470 | mov loc1 = b0 | ||
1471 | mov out0 = b0 | ||
1472 | mov loc2 = r8 | ||
1473 | mov loc3 = r15 | ||
1474 | ;; | ||
1475 | adds out0 = -MCOUNT_INSN_SIZE, out0 | ||
1476 | mov out1 = in2 | ||
1477 | mov b6 = r3 | ||
1478 | |||
1479 | br.call.sptk.many b0 = b6 | ||
1480 | ;; | ||
1481 | mov ar.pfs = loc0 | ||
1482 | mov b0 = loc1 | ||
1483 | mov r8 = loc2 | ||
1484 | mov r15 = loc3 | ||
1485 | br ftrace_stub | ||
1486 | ;; | ||
1487 | END(_mcount) | ||
1488 | #endif | ||
1489 | |||
1490 | GLOBAL_ENTRY(ftrace_stub) | ||
1491 | mov r3 = b0 | ||
1492 | movl r2 = _mcount_ret_helper | ||
1493 | ;; | ||
1494 | mov b6 = r2 | ||
1495 | mov b7 = r3 | ||
1496 | br.ret.sptk.many b6 | ||
1497 | |||
1498 | _mcount_ret_helper: | ||
1499 | mov b0 = r42 | ||
1500 | mov r1 = r41 | ||
1501 | mov ar.pfs = r40 | ||
1502 | br b7 | ||
1503 | END(ftrace_stub) | ||
1504 | |||
1505 | #endif /* CONFIG_FUNCTION_TRACER */ | ||
1506 | |||
1407 | .rodata | 1507 | .rodata |
1408 | .align 8 | 1508 | .align 8 |
1409 | .globl sys_call_table | 1509 | .globl sys_call_table |
diff --git a/arch/ia64/kernel/ftrace.c b/arch/ia64/kernel/ftrace.c new file mode 100644 index 000000000000..7fc8c961b1f7 --- /dev/null +++ b/arch/ia64/kernel/ftrace.c | |||
@@ -0,0 +1,206 @@ | |||
1 | /* | ||
2 | * Dynamic function tracing support. | ||
3 | * | ||
4 | * Copyright (C) 2008 Shaohua Li <shaohua.li@intel.com> | ||
5 | * | ||
6 | * For licencing details, see COPYING. | ||
7 | * | ||
8 | * Defines low-level handling of mcount calls when the kernel | ||
9 | * is compiled with the -pg flag. When using dynamic ftrace, the | ||
10 | * mcount call-sites get patched lazily with NOP till they are | ||
11 | * enabled. All code mutation routines here take effect atomically. | ||
12 | */ | ||
13 | |||
14 | #include <linux/uaccess.h> | ||
15 | #include <linux/ftrace.h> | ||
16 | |||
17 | #include <asm/cacheflush.h> | ||
18 | #include <asm/patch.h> | ||
19 | |||
20 | /* In IA64, each function will be added below two bundles with -pg option */ | ||
21 | static unsigned char __attribute__((aligned(8))) | ||
22 | ftrace_orig_code[MCOUNT_INSN_SIZE] = { | ||
23 | 0x02, 0x40, 0x31, 0x10, 0x80, 0x05, /* alloc r40=ar.pfs,12,8,0 */ | ||
24 | 0xb0, 0x02, 0x00, 0x00, 0x42, 0x40, /* mov r43=r0;; */ | ||
25 | 0x05, 0x00, 0xc4, 0x00, /* mov r42=b0 */ | ||
26 | 0x11, 0x48, 0x01, 0x02, 0x00, 0x21, /* mov r41=r1 */ | ||
27 | 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /* nop.i 0x0 */ | ||
28 | 0x08, 0x00, 0x00, 0x50 /* br.call.sptk.many b0 = _mcount;; */ | ||
29 | }; | ||
30 | |||
31 | struct ftrace_orig_insn { | ||
32 | u64 dummy1, dummy2, dummy3; | ||
33 | u64 dummy4:64-41+13; | ||
34 | u64 imm20:20; | ||
35 | u64 dummy5:3; | ||
36 | u64 sign:1; | ||
37 | u64 dummy6:4; | ||
38 | }; | ||
39 | |||
40 | /* mcount stub will be converted below for nop */ | ||
41 | static unsigned char ftrace_nop_code[MCOUNT_INSN_SIZE] = { | ||
42 | 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MII] nop.m 0x0 */ | ||
43 | 0x30, 0x00, 0x00, 0x60, 0x00, 0x00, /* mov r3=ip */ | ||
44 | 0x00, 0x00, 0x04, 0x00, /* nop.i 0x0 */ | ||
45 | 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0x0 */ | ||
46 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* nop.x 0x0;; */ | ||
47 | 0x00, 0x00, 0x04, 0x00 | ||
48 | }; | ||
49 | |||
50 | static unsigned char *ftrace_nop_replace(void) | ||
51 | { | ||
52 | return ftrace_nop_code; | ||
53 | } | ||
54 | |||
55 | /* | ||
56 | * mcount stub will be converted below for call | ||
57 | * Note: Just the last instruction is changed against nop | ||
58 | * */ | ||
59 | static unsigned char __attribute__((aligned(8))) | ||
60 | ftrace_call_code[MCOUNT_INSN_SIZE] = { | ||
61 | 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MII] nop.m 0x0 */ | ||
62 | 0x30, 0x00, 0x00, 0x60, 0x00, 0x00, /* mov r3=ip */ | ||
63 | 0x00, 0x00, 0x04, 0x00, /* nop.i 0x0 */ | ||
64 | 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0x0 */ | ||
65 | 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, /* brl.many .;;*/ | ||
66 | 0xf8, 0xff, 0xff, 0xc8 | ||
67 | }; | ||
68 | |||
69 | struct ftrace_call_insn { | ||
70 | u64 dummy1, dummy2; | ||
71 | u64 dummy3:48; | ||
72 | u64 imm39_l:16; | ||
73 | u64 imm39_h:23; | ||
74 | u64 dummy4:13; | ||
75 | u64 imm20:20; | ||
76 | u64 dummy5:3; | ||
77 | u64 i:1; | ||
78 | u64 dummy6:4; | ||
79 | }; | ||
80 | |||
81 | static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) | ||
82 | { | ||
83 | struct ftrace_call_insn *code = (void *)ftrace_call_code; | ||
84 | unsigned long offset = addr - (ip + 0x10); | ||
85 | |||
86 | code->imm39_l = offset >> 24; | ||
87 | code->imm39_h = offset >> 40; | ||
88 | code->imm20 = offset >> 4; | ||
89 | code->i = offset >> 63; | ||
90 | return ftrace_call_code; | ||
91 | } | ||
92 | |||
93 | static int | ||
94 | ftrace_modify_code(unsigned long ip, unsigned char *old_code, | ||
95 | unsigned char *new_code, int do_check) | ||
96 | { | ||
97 | unsigned char replaced[MCOUNT_INSN_SIZE]; | ||
98 | |||
99 | /* | ||
100 | * Note: Due to modules and __init, code can | ||
101 | * disappear and change, we need to protect against faulting | ||
102 | * as well as code changing. We do this by using the | ||
103 | * probe_kernel_* functions. | ||
104 | * | ||
105 | * No real locking needed, this code is run through | ||
106 | * kstop_machine, or before SMP starts. | ||
107 | */ | ||
108 | |||
109 | if (!do_check) | ||
110 | goto skip_check; | ||
111 | |||
112 | /* read the text we want to modify */ | ||
113 | if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE)) | ||
114 | return -EFAULT; | ||
115 | |||
116 | /* Make sure it is what we expect it to be */ | ||
117 | if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0) | ||
118 | return -EINVAL; | ||
119 | |||
120 | skip_check: | ||
121 | /* replace the text with the new text */ | ||
122 | if (probe_kernel_write(((void *)ip), new_code, MCOUNT_INSN_SIZE)) | ||
123 | return -EPERM; | ||
124 | flush_icache_range(ip, ip + MCOUNT_INSN_SIZE); | ||
125 | |||
126 | return 0; | ||
127 | } | ||
128 | |||
129 | static int ftrace_make_nop_check(struct dyn_ftrace *rec, unsigned long addr) | ||
130 | { | ||
131 | unsigned char __attribute__((aligned(8))) replaced[MCOUNT_INSN_SIZE]; | ||
132 | unsigned long ip = rec->ip; | ||
133 | |||
134 | if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE)) | ||
135 | return -EFAULT; | ||
136 | if (rec->flags & FTRACE_FL_CONVERTED) { | ||
137 | struct ftrace_call_insn *call_insn, *tmp_call; | ||
138 | |||
139 | call_insn = (void *)ftrace_call_code; | ||
140 | tmp_call = (void *)replaced; | ||
141 | call_insn->imm39_l = tmp_call->imm39_l; | ||
142 | call_insn->imm39_h = tmp_call->imm39_h; | ||
143 | call_insn->imm20 = tmp_call->imm20; | ||
144 | call_insn->i = tmp_call->i; | ||
145 | if (memcmp(replaced, ftrace_call_code, MCOUNT_INSN_SIZE) != 0) | ||
146 | return -EINVAL; | ||
147 | return 0; | ||
148 | } else { | ||
149 | struct ftrace_orig_insn *call_insn, *tmp_call; | ||
150 | |||
151 | call_insn = (void *)ftrace_orig_code; | ||
152 | tmp_call = (void *)replaced; | ||
153 | call_insn->sign = tmp_call->sign; | ||
154 | call_insn->imm20 = tmp_call->imm20; | ||
155 | if (memcmp(replaced, ftrace_orig_code, MCOUNT_INSN_SIZE) != 0) | ||
156 | return -EINVAL; | ||
157 | return 0; | ||
158 | } | ||
159 | } | ||
160 | |||
161 | int ftrace_make_nop(struct module *mod, | ||
162 | struct dyn_ftrace *rec, unsigned long addr) | ||
163 | { | ||
164 | int ret; | ||
165 | char *new; | ||
166 | |||
167 | ret = ftrace_make_nop_check(rec, addr); | ||
168 | if (ret) | ||
169 | return ret; | ||
170 | new = ftrace_nop_replace(); | ||
171 | return ftrace_modify_code(rec->ip, NULL, new, 0); | ||
172 | } | ||
173 | |||
174 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) | ||
175 | { | ||
176 | unsigned long ip = rec->ip; | ||
177 | unsigned char *old, *new; | ||
178 | |||
179 | old= ftrace_nop_replace(); | ||
180 | new = ftrace_call_replace(ip, addr); | ||
181 | return ftrace_modify_code(ip, old, new, 1); | ||
182 | } | ||
183 | |||
184 | /* in IA64, _mcount can't directly call ftrace_stub. Only jump is ok */ | ||
185 | int ftrace_update_ftrace_func(ftrace_func_t func) | ||
186 | { | ||
187 | unsigned long ip; | ||
188 | unsigned long addr = ((struct fnptr *)ftrace_call)->ip; | ||
189 | |||
190 | if (func == ftrace_stub) | ||
191 | return 0; | ||
192 | ip = ((struct fnptr *)func)->ip; | ||
193 | |||
194 | ia64_patch_imm64(addr + 2, ip); | ||
195 | |||
196 | flush_icache_range(addr, addr + 16); | ||
197 | return 0; | ||
198 | } | ||
199 | |||
200 | /* run from kstop_machine */ | ||
201 | int __init ftrace_dyn_arch_init(void *data) | ||
202 | { | ||
203 | *(unsigned long *)data = 0; | ||
204 | |||
205 | return 0; | ||
206 | } | ||
diff --git a/arch/ia64/kernel/ia64_ksyms.c b/arch/ia64/kernel/ia64_ksyms.c index 6da1f20d7372..2d311864e359 100644 --- a/arch/ia64/kernel/ia64_ksyms.c +++ b/arch/ia64/kernel/ia64_ksyms.c | |||
@@ -112,3 +112,9 @@ EXPORT_SYMBOL_GPL(esi_call_phys); | |||
112 | #endif | 112 | #endif |
113 | extern char ia64_ivt[]; | 113 | extern char ia64_ivt[]; |
114 | EXPORT_SYMBOL(ia64_ivt); | 114 | EXPORT_SYMBOL(ia64_ivt); |
115 | |||
116 | #include <asm/ftrace.h> | ||
117 | #ifdef CONFIG_FUNCTION_TRACER | ||
118 | /* mcount is defined in assembly */ | ||
119 | EXPORT_SYMBOL(_mcount); | ||
120 | #endif | ||
diff --git a/arch/m68k/include/asm/ftrace.h b/arch/m68k/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/m68k/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||
diff --git a/arch/mips/include/asm/ftrace.h b/arch/mips/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/mips/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||
diff --git a/arch/parisc/include/asm/ftrace.h b/arch/parisc/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/parisc/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||
diff --git a/arch/um/include/asm/ftrace.h b/arch/um/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/um/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 748e50a1a152..0885245e6808 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig | |||
@@ -34,6 +34,8 @@ config X86 | |||
34 | select HAVE_FUNCTION_TRACER | 34 | select HAVE_FUNCTION_TRACER |
35 | select HAVE_FUNCTION_GRAPH_TRACER | 35 | select HAVE_FUNCTION_GRAPH_TRACER |
36 | select HAVE_FUNCTION_TRACE_MCOUNT_TEST | 36 | select HAVE_FUNCTION_TRACE_MCOUNT_TEST |
37 | select HAVE_FTRACE_NMI_ENTER if DYNAMIC_FTRACE | ||
38 | select HAVE_FTRACE_SYSCALLS | ||
37 | select HAVE_KVM | 39 | select HAVE_KVM |
38 | select HAVE_ARCH_KGDB | 40 | select HAVE_ARCH_KGDB |
39 | select HAVE_ARCH_TRACEHOOK | 41 | select HAVE_ARCH_TRACEHOOK |
diff --git a/arch/x86/include/asm/cacheflush.h b/arch/x86/include/asm/cacheflush.h index b3894bf52fcd..e55dfc1ad453 100644 --- a/arch/x86/include/asm/cacheflush.h +++ b/arch/x86/include/asm/cacheflush.h | |||
@@ -126,6 +126,11 @@ void clflush_cache_range(void *addr, unsigned int size); | |||
126 | #ifdef CONFIG_DEBUG_RODATA | 126 | #ifdef CONFIG_DEBUG_RODATA |
127 | void mark_rodata_ro(void); | 127 | void mark_rodata_ro(void); |
128 | extern const int rodata_test_data; | 128 | extern const int rodata_test_data; |
129 | void set_kernel_text_rw(void); | ||
130 | void set_kernel_text_ro(void); | ||
131 | #else | ||
132 | static inline void set_kernel_text_rw(void) { } | ||
133 | static inline void set_kernel_text_ro(void) { } | ||
129 | #endif | 134 | #endif |
130 | 135 | ||
131 | #ifdef CONFIG_DEBUG_RODATA_TEST | 136 | #ifdef CONFIG_DEBUG_RODATA_TEST |
diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h index 63a79c77d220..81937a5dc77c 100644 --- a/arch/x86/include/asm/fixmap.h +++ b/arch/x86/include/asm/fixmap.h | |||
@@ -111,6 +111,8 @@ enum fixed_addresses { | |||
111 | #ifdef CONFIG_PARAVIRT | 111 | #ifdef CONFIG_PARAVIRT |
112 | FIX_PARAVIRT_BOOTMAP, | 112 | FIX_PARAVIRT_BOOTMAP, |
113 | #endif | 113 | #endif |
114 | FIX_TEXT_POKE0, /* reserve 2 pages for text_poke() */ | ||
115 | FIX_TEXT_POKE1, | ||
114 | __end_of_permanent_fixed_addresses, | 116 | __end_of_permanent_fixed_addresses, |
115 | #ifdef CONFIG_PROVIDE_OHCI1394_DMA_INIT | 117 | #ifdef CONFIG_PROVIDE_OHCI1394_DMA_INIT |
116 | FIX_OHCI1394_BASE, | 118 | FIX_OHCI1394_BASE, |
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index db24c2278be0..bd2c6511c887 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h | |||
@@ -28,6 +28,13 @@ | |||
28 | 28 | ||
29 | #endif | 29 | #endif |
30 | 30 | ||
31 | /* FIXME: I don't want to stay hardcoded */ | ||
32 | #ifdef CONFIG_X86_64 | ||
33 | # define FTRACE_SYSCALL_MAX 296 | ||
34 | #else | ||
35 | # define FTRACE_SYSCALL_MAX 333 | ||
36 | #endif | ||
37 | |||
31 | #ifdef CONFIG_FUNCTION_TRACER | 38 | #ifdef CONFIG_FUNCTION_TRACER |
32 | #define MCOUNT_ADDR ((long)(mcount)) | 39 | #define MCOUNT_ADDR ((long)(mcount)) |
33 | #define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */ | 40 | #define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */ |
diff --git a/arch/x86/include/asm/ptrace-abi.h b/arch/x86/include/asm/ptrace-abi.h index 8e0f8d199e05..86723035a515 100644 --- a/arch/x86/include/asm/ptrace-abi.h +++ b/arch/x86/include/asm/ptrace-abi.h | |||
@@ -80,8 +80,6 @@ | |||
80 | 80 | ||
81 | #define PTRACE_SINGLEBLOCK 33 /* resume execution until next branch */ | 81 | #define PTRACE_SINGLEBLOCK 33 /* resume execution until next branch */ |
82 | 82 | ||
83 | #ifdef CONFIG_X86_PTRACE_BTS | ||
84 | |||
85 | #ifndef __ASSEMBLY__ | 83 | #ifndef __ASSEMBLY__ |
86 | #include <linux/types.h> | 84 | #include <linux/types.h> |
87 | 85 | ||
@@ -140,6 +138,5 @@ struct ptrace_bts_config { | |||
140 | BTS records are read from oldest to newest. | 138 | BTS records are read from oldest to newest. |
141 | Returns number of BTS records drained. | 139 | Returns number of BTS records drained. |
142 | */ | 140 | */ |
143 | #endif /* CONFIG_X86_PTRACE_BTS */ | ||
144 | 141 | ||
145 | #endif /* _ASM_X86_PTRACE_ABI_H */ | 142 | #endif /* _ASM_X86_PTRACE_ABI_H */ |
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index df9d5f78385e..8820a73ae090 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h | |||
@@ -94,6 +94,7 @@ struct thread_info { | |||
94 | #define TIF_FORCED_TF 24 /* true if TF in eflags artificially */ | 94 | #define TIF_FORCED_TF 24 /* true if TF in eflags artificially */ |
95 | #define TIF_DEBUGCTLMSR 25 /* uses thread_struct.debugctlmsr */ | 95 | #define TIF_DEBUGCTLMSR 25 /* uses thread_struct.debugctlmsr */ |
96 | #define TIF_DS_AREA_MSR 26 /* uses thread_struct.ds_area_msr */ | 96 | #define TIF_DS_AREA_MSR 26 /* uses thread_struct.ds_area_msr */ |
97 | #define TIF_SYSCALL_FTRACE 27 /* for ftrace syscall instrumentation */ | ||
97 | 98 | ||
98 | #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) | 99 | #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) |
99 | #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) | 100 | #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) |
@@ -115,15 +116,17 @@ struct thread_info { | |||
115 | #define _TIF_FORCED_TF (1 << TIF_FORCED_TF) | 116 | #define _TIF_FORCED_TF (1 << TIF_FORCED_TF) |
116 | #define _TIF_DEBUGCTLMSR (1 << TIF_DEBUGCTLMSR) | 117 | #define _TIF_DEBUGCTLMSR (1 << TIF_DEBUGCTLMSR) |
117 | #define _TIF_DS_AREA_MSR (1 << TIF_DS_AREA_MSR) | 118 | #define _TIF_DS_AREA_MSR (1 << TIF_DS_AREA_MSR) |
119 | #define _TIF_SYSCALL_FTRACE (1 << TIF_SYSCALL_FTRACE) | ||
118 | 120 | ||
119 | /* work to do in syscall_trace_enter() */ | 121 | /* work to do in syscall_trace_enter() */ |
120 | #define _TIF_WORK_SYSCALL_ENTRY \ | 122 | #define _TIF_WORK_SYSCALL_ENTRY \ |
121 | (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | \ | 123 | (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_FTRACE | \ |
122 | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | _TIF_SINGLESTEP) | 124 | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | _TIF_SINGLESTEP) |
123 | 125 | ||
124 | /* work to do in syscall_trace_leave() */ | 126 | /* work to do in syscall_trace_leave() */ |
125 | #define _TIF_WORK_SYSCALL_EXIT \ | 127 | #define _TIF_WORK_SYSCALL_EXIT \ |
126 | (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SINGLESTEP) | 128 | (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SINGLESTEP | \ |
129 | _TIF_SYSCALL_FTRACE) | ||
127 | 130 | ||
128 | /* work to do on interrupt/exception return */ | 131 | /* work to do on interrupt/exception return */ |
129 | #define _TIF_WORK_MASK \ | 132 | #define _TIF_WORK_MASK \ |
@@ -132,7 +135,7 @@ struct thread_info { | |||
132 | _TIF_SINGLESTEP|_TIF_SECCOMP|_TIF_SYSCALL_EMU)) | 135 | _TIF_SINGLESTEP|_TIF_SECCOMP|_TIF_SYSCALL_EMU)) |
133 | 136 | ||
134 | /* work to do on any return to user space */ | 137 | /* work to do on any return to user space */ |
135 | #define _TIF_ALLWORK_MASK (0x0000FFFF & ~_TIF_SECCOMP) | 138 | #define _TIF_ALLWORK_MASK ((0x0000FFFF & ~_TIF_SECCOMP) | _TIF_SYSCALL_FTRACE) |
136 | 139 | ||
137 | /* Only used for 64 bit */ | 140 | /* Only used for 64 bit */ |
138 | #define _TIF_DO_NOTIFY_MASK \ | 141 | #define _TIF_DO_NOTIFY_MASK \ |
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index c611ad64137f..145cce75cda7 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile | |||
@@ -66,7 +66,8 @@ obj-$(CONFIG_X86_MPPARSE) += mpparse.o | |||
66 | obj-y += apic/ | 66 | obj-y += apic/ |
67 | obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups_32.o | 67 | obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups_32.o |
68 | obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o | 68 | obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o |
69 | obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o | 69 | obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o |
70 | obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o | ||
70 | obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o | 71 | obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o |
71 | obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o | 72 | obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o |
72 | obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o | 73 | obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o |
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 4c80f1557433..f57658702571 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c | |||
@@ -5,6 +5,7 @@ | |||
5 | #include <linux/kprobes.h> | 5 | #include <linux/kprobes.h> |
6 | #include <linux/mm.h> | 6 | #include <linux/mm.h> |
7 | #include <linux/vmalloc.h> | 7 | #include <linux/vmalloc.h> |
8 | #include <linux/memory.h> | ||
8 | #include <asm/alternative.h> | 9 | #include <asm/alternative.h> |
9 | #include <asm/sections.h> | 10 | #include <asm/sections.h> |
10 | #include <asm/pgtable.h> | 11 | #include <asm/pgtable.h> |
@@ -12,7 +13,9 @@ | |||
12 | #include <asm/nmi.h> | 13 | #include <asm/nmi.h> |
13 | #include <asm/vsyscall.h> | 14 | #include <asm/vsyscall.h> |
14 | #include <asm/cacheflush.h> | 15 | #include <asm/cacheflush.h> |
16 | #include <asm/tlbflush.h> | ||
15 | #include <asm/io.h> | 17 | #include <asm/io.h> |
18 | #include <asm/fixmap.h> | ||
16 | 19 | ||
17 | #define MAX_PATCH_LEN (255-1) | 20 | #define MAX_PATCH_LEN (255-1) |
18 | 21 | ||
@@ -226,6 +229,7 @@ static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end) | |||
226 | { | 229 | { |
227 | u8 **ptr; | 230 | u8 **ptr; |
228 | 231 | ||
232 | mutex_lock(&text_mutex); | ||
229 | for (ptr = start; ptr < end; ptr++) { | 233 | for (ptr = start; ptr < end; ptr++) { |
230 | if (*ptr < text) | 234 | if (*ptr < text) |
231 | continue; | 235 | continue; |
@@ -234,6 +238,7 @@ static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end) | |||
234 | /* turn DS segment override prefix into lock prefix */ | 238 | /* turn DS segment override prefix into lock prefix */ |
235 | text_poke(*ptr, ((unsigned char []){0xf0}), 1); | 239 | text_poke(*ptr, ((unsigned char []){0xf0}), 1); |
236 | }; | 240 | }; |
241 | mutex_unlock(&text_mutex); | ||
237 | } | 242 | } |
238 | 243 | ||
239 | static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end) | 244 | static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end) |
@@ -243,6 +248,7 @@ static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end | |||
243 | if (noreplace_smp) | 248 | if (noreplace_smp) |
244 | return; | 249 | return; |
245 | 250 | ||
251 | mutex_lock(&text_mutex); | ||
246 | for (ptr = start; ptr < end; ptr++) { | 252 | for (ptr = start; ptr < end; ptr++) { |
247 | if (*ptr < text) | 253 | if (*ptr < text) |
248 | continue; | 254 | continue; |
@@ -251,6 +257,7 @@ static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end | |||
251 | /* turn lock prefix into DS segment override prefix */ | 257 | /* turn lock prefix into DS segment override prefix */ |
252 | text_poke(*ptr, ((unsigned char []){0x3E}), 1); | 258 | text_poke(*ptr, ((unsigned char []){0x3E}), 1); |
253 | }; | 259 | }; |
260 | mutex_unlock(&text_mutex); | ||
254 | } | 261 | } |
255 | 262 | ||
256 | struct smp_alt_module { | 263 | struct smp_alt_module { |
@@ -500,15 +507,16 @@ void *text_poke_early(void *addr, const void *opcode, size_t len) | |||
500 | * It means the size must be writable atomically and the address must be aligned | 507 | * It means the size must be writable atomically and the address must be aligned |
501 | * in a way that permits an atomic write. It also makes sure we fit on a single | 508 | * in a way that permits an atomic write. It also makes sure we fit on a single |
502 | * page. | 509 | * page. |
510 | * | ||
511 | * Note: Must be called under text_mutex. | ||
503 | */ | 512 | */ |
504 | void *__kprobes text_poke(void *addr, const void *opcode, size_t len) | 513 | void *__kprobes text_poke(void *addr, const void *opcode, size_t len) |
505 | { | 514 | { |
515 | unsigned long flags; | ||
506 | char *vaddr; | 516 | char *vaddr; |
507 | int nr_pages = 2; | ||
508 | struct page *pages[2]; | 517 | struct page *pages[2]; |
509 | int i; | 518 | int i; |
510 | 519 | ||
511 | might_sleep(); | ||
512 | if (!core_kernel_text((unsigned long)addr)) { | 520 | if (!core_kernel_text((unsigned long)addr)) { |
513 | pages[0] = vmalloc_to_page(addr); | 521 | pages[0] = vmalloc_to_page(addr); |
514 | pages[1] = vmalloc_to_page(addr + PAGE_SIZE); | 522 | pages[1] = vmalloc_to_page(addr + PAGE_SIZE); |
@@ -518,18 +526,21 @@ void *__kprobes text_poke(void *addr, const void *opcode, size_t len) | |||
518 | pages[1] = virt_to_page(addr + PAGE_SIZE); | 526 | pages[1] = virt_to_page(addr + PAGE_SIZE); |
519 | } | 527 | } |
520 | BUG_ON(!pages[0]); | 528 | BUG_ON(!pages[0]); |
521 | if (!pages[1]) | 529 | local_irq_save(flags); |
522 | nr_pages = 1; | 530 | set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0])); |
523 | vaddr = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL); | 531 | if (pages[1]) |
524 | BUG_ON(!vaddr); | 532 | set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1])); |
525 | local_irq_disable(); | 533 | vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0); |
526 | memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len); | 534 | memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len); |
527 | local_irq_enable(); | 535 | clear_fixmap(FIX_TEXT_POKE0); |
528 | vunmap(vaddr); | 536 | if (pages[1]) |
537 | clear_fixmap(FIX_TEXT_POKE1); | ||
538 | local_flush_tlb(); | ||
529 | sync_core(); | 539 | sync_core(); |
530 | /* Could also do a CLFLUSH here to speed up CPU recovery; but | 540 | /* Could also do a CLFLUSH here to speed up CPU recovery; but |
531 | that causes hangs on some VIA CPUs. */ | 541 | that causes hangs on some VIA CPUs. */ |
532 | for (i = 0; i < len; i++) | 542 | for (i = 0; i < len; i++) |
533 | BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]); | 543 | BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]); |
544 | local_irq_restore(flags); | ||
534 | return addr; | 545 | return addr; |
535 | } | 546 | } |
diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c index 23da96e57b17..05209b5cc6ca 100644 --- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c +++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c | |||
@@ -33,7 +33,7 @@ | |||
33 | #include <linux/cpufreq.h> | 33 | #include <linux/cpufreq.h> |
34 | #include <linux/compiler.h> | 34 | #include <linux/compiler.h> |
35 | #include <linux/dmi.h> | 35 | #include <linux/dmi.h> |
36 | #include <linux/ftrace.h> | 36 | #include <trace/power.h> |
37 | 37 | ||
38 | #include <linux/acpi.h> | 38 | #include <linux/acpi.h> |
39 | #include <linux/io.h> | 39 | #include <linux/io.h> |
@@ -72,6 +72,8 @@ struct acpi_cpufreq_data { | |||
72 | 72 | ||
73 | static DEFINE_PER_CPU(struct acpi_cpufreq_data *, drv_data); | 73 | static DEFINE_PER_CPU(struct acpi_cpufreq_data *, drv_data); |
74 | 74 | ||
75 | DEFINE_TRACE(power_mark); | ||
76 | |||
75 | /* acpi_perf_data is a pointer to percpu data. */ | 77 | /* acpi_perf_data is a pointer to percpu data. */ |
76 | static struct acpi_processor_performance *acpi_perf_data; | 78 | static struct acpi_processor_performance *acpi_perf_data; |
77 | 79 | ||
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index dd2130b0fb3e..95ea5fa7d444 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c | |||
@@ -15,6 +15,7 @@ | |||
15 | #include <linux/bug.h> | 15 | #include <linux/bug.h> |
16 | #include <linux/nmi.h> | 16 | #include <linux/nmi.h> |
17 | #include <linux/sysfs.h> | 17 | #include <linux/sysfs.h> |
18 | #include <linux/ftrace.h> | ||
18 | 19 | ||
19 | #include <asm/stacktrace.h> | 20 | #include <asm/stacktrace.h> |
20 | 21 | ||
@@ -196,6 +197,11 @@ unsigned __kprobes long oops_begin(void) | |||
196 | int cpu; | 197 | int cpu; |
197 | unsigned long flags; | 198 | unsigned long flags; |
198 | 199 | ||
200 | /* notify the hw-branch tracer so it may disable tracing and | ||
201 | add the last trace to the trace buffer - | ||
202 | the earlier this happens, the more useful the trace. */ | ||
203 | trace_hw_branch_oops(); | ||
204 | |||
199 | oops_enter(); | 205 | oops_enter(); |
200 | 206 | ||
201 | /* racy, but better than risking deadlock. */ | 207 | /* racy, but better than risking deadlock. */ |
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 76f7141e0f91..61df77532120 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c | |||
@@ -18,6 +18,7 @@ | |||
18 | #include <linux/init.h> | 18 | #include <linux/init.h> |
19 | #include <linux/list.h> | 19 | #include <linux/list.h> |
20 | 20 | ||
21 | #include <asm/cacheflush.h> | ||
21 | #include <asm/ftrace.h> | 22 | #include <asm/ftrace.h> |
22 | #include <linux/ftrace.h> | 23 | #include <linux/ftrace.h> |
23 | #include <asm/nops.h> | 24 | #include <asm/nops.h> |
@@ -26,6 +27,18 @@ | |||
26 | 27 | ||
27 | #ifdef CONFIG_DYNAMIC_FTRACE | 28 | #ifdef CONFIG_DYNAMIC_FTRACE |
28 | 29 | ||
30 | int ftrace_arch_code_modify_prepare(void) | ||
31 | { | ||
32 | set_kernel_text_rw(); | ||
33 | return 0; | ||
34 | } | ||
35 | |||
36 | int ftrace_arch_code_modify_post_process(void) | ||
37 | { | ||
38 | set_kernel_text_ro(); | ||
39 | return 0; | ||
40 | } | ||
41 | |||
29 | union ftrace_code_union { | 42 | union ftrace_code_union { |
30 | char code[MCOUNT_INSN_SIZE]; | 43 | char code[MCOUNT_INSN_SIZE]; |
31 | struct { | 44 | struct { |
@@ -66,11 +79,11 @@ static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) | |||
66 | * | 79 | * |
67 | * 1) Put the instruction pointer into the IP buffer | 80 | * 1) Put the instruction pointer into the IP buffer |
68 | * and the new code into the "code" buffer. | 81 | * and the new code into the "code" buffer. |
69 | * 2) Set a flag that says we are modifying code | 82 | * 2) Wait for any running NMIs to finish and set a flag that says |
70 | * 3) Wait for any running NMIs to finish. | 83 | * we are modifying code, it is done in an atomic operation. |
71 | * 4) Write the code | 84 | * 3) Write the code |
72 | * 5) clear the flag. | 85 | * 4) clear the flag. |
73 | * 6) Wait for any running NMIs to finish. | 86 | * 5) Wait for any running NMIs to finish. |
74 | * | 87 | * |
75 | * If an NMI is executed, the first thing it does is to call | 88 | * If an NMI is executed, the first thing it does is to call |
76 | * "ftrace_nmi_enter". This will check if the flag is set to write | 89 | * "ftrace_nmi_enter". This will check if the flag is set to write |
@@ -82,9 +95,9 @@ static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) | |||
82 | * are the same as what exists. | 95 | * are the same as what exists. |
83 | */ | 96 | */ |
84 | 97 | ||
85 | static atomic_t in_nmi = ATOMIC_INIT(0); | 98 | #define MOD_CODE_WRITE_FLAG (1 << 31) /* set when NMI should do the write */ |
99 | static atomic_t nmi_running = ATOMIC_INIT(0); | ||
86 | static int mod_code_status; /* holds return value of text write */ | 100 | static int mod_code_status; /* holds return value of text write */ |
87 | static int mod_code_write; /* set when NMI should do the write */ | ||
88 | static void *mod_code_ip; /* holds the IP to write to */ | 101 | static void *mod_code_ip; /* holds the IP to write to */ |
89 | static void *mod_code_newcode; /* holds the text to write to the IP */ | 102 | static void *mod_code_newcode; /* holds the text to write to the IP */ |
90 | 103 | ||
@@ -101,6 +114,20 @@ int ftrace_arch_read_dyn_info(char *buf, int size) | |||
101 | return r; | 114 | return r; |
102 | } | 115 | } |
103 | 116 | ||
117 | static void clear_mod_flag(void) | ||
118 | { | ||
119 | int old = atomic_read(&nmi_running); | ||
120 | |||
121 | for (;;) { | ||
122 | int new = old & ~MOD_CODE_WRITE_FLAG; | ||
123 | |||
124 | if (old == new) | ||
125 | break; | ||
126 | |||
127 | old = atomic_cmpxchg(&nmi_running, old, new); | ||
128 | } | ||
129 | } | ||
130 | |||
104 | static void ftrace_mod_code(void) | 131 | static void ftrace_mod_code(void) |
105 | { | 132 | { |
106 | /* | 133 | /* |
@@ -111,37 +138,52 @@ static void ftrace_mod_code(void) | |||
111 | */ | 138 | */ |
112 | mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode, | 139 | mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode, |
113 | MCOUNT_INSN_SIZE); | 140 | MCOUNT_INSN_SIZE); |
141 | |||
142 | /* if we fail, then kill any new writers */ | ||
143 | if (mod_code_status) | ||
144 | clear_mod_flag(); | ||
114 | } | 145 | } |
115 | 146 | ||
116 | void ftrace_nmi_enter(void) | 147 | void ftrace_nmi_enter(void) |
117 | { | 148 | { |
118 | atomic_inc(&in_nmi); | 149 | if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) { |
119 | /* Must have in_nmi seen before reading write flag */ | 150 | smp_rmb(); |
120 | smp_mb(); | ||
121 | if (mod_code_write) { | ||
122 | ftrace_mod_code(); | 151 | ftrace_mod_code(); |
123 | atomic_inc(&nmi_update_count); | 152 | atomic_inc(&nmi_update_count); |
124 | } | 153 | } |
154 | /* Must have previous changes seen before executions */ | ||
155 | smp_mb(); | ||
125 | } | 156 | } |
126 | 157 | ||
127 | void ftrace_nmi_exit(void) | 158 | void ftrace_nmi_exit(void) |
128 | { | 159 | { |
129 | /* Finish all executions before clearing in_nmi */ | 160 | /* Finish all executions before clearing nmi_running */ |
130 | smp_wmb(); | 161 | smp_mb(); |
131 | atomic_dec(&in_nmi); | 162 | atomic_dec(&nmi_running); |
163 | } | ||
164 | |||
165 | static void wait_for_nmi_and_set_mod_flag(void) | ||
166 | { | ||
167 | if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG)) | ||
168 | return; | ||
169 | |||
170 | do { | ||
171 | cpu_relax(); | ||
172 | } while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG)); | ||
173 | |||
174 | nmi_wait_count++; | ||
132 | } | 175 | } |
133 | 176 | ||
134 | static void wait_for_nmi(void) | 177 | static void wait_for_nmi(void) |
135 | { | 178 | { |
136 | int waited = 0; | 179 | if (!atomic_read(&nmi_running)) |
180 | return; | ||
137 | 181 | ||
138 | while (atomic_read(&in_nmi)) { | 182 | do { |
139 | waited = 1; | ||
140 | cpu_relax(); | 183 | cpu_relax(); |
141 | } | 184 | } while (atomic_read(&nmi_running)); |
142 | 185 | ||
143 | if (waited) | 186 | nmi_wait_count++; |
144 | nmi_wait_count++; | ||
145 | } | 187 | } |
146 | 188 | ||
147 | static int | 189 | static int |
@@ -151,14 +193,9 @@ do_ftrace_mod_code(unsigned long ip, void *new_code) | |||
151 | mod_code_newcode = new_code; | 193 | mod_code_newcode = new_code; |
152 | 194 | ||
153 | /* The buffers need to be visible before we let NMIs write them */ | 195 | /* The buffers need to be visible before we let NMIs write them */ |
154 | smp_wmb(); | ||
155 | |||
156 | mod_code_write = 1; | ||
157 | |||
158 | /* Make sure write bit is visible before we wait on NMIs */ | ||
159 | smp_mb(); | 196 | smp_mb(); |
160 | 197 | ||
161 | wait_for_nmi(); | 198 | wait_for_nmi_and_set_mod_flag(); |
162 | 199 | ||
163 | /* Make sure all running NMIs have finished before we write the code */ | 200 | /* Make sure all running NMIs have finished before we write the code */ |
164 | smp_mb(); | 201 | smp_mb(); |
@@ -166,13 +203,9 @@ do_ftrace_mod_code(unsigned long ip, void *new_code) | |||
166 | ftrace_mod_code(); | 203 | ftrace_mod_code(); |
167 | 204 | ||
168 | /* Make sure the write happens before clearing the bit */ | 205 | /* Make sure the write happens before clearing the bit */ |
169 | smp_wmb(); | ||
170 | |||
171 | mod_code_write = 0; | ||
172 | |||
173 | /* make sure NMIs see the cleared bit */ | ||
174 | smp_mb(); | 206 | smp_mb(); |
175 | 207 | ||
208 | clear_mod_flag(); | ||
176 | wait_for_nmi(); | 209 | wait_for_nmi(); |
177 | 210 | ||
178 | return mod_code_status; | 211 | return mod_code_status; |
@@ -368,25 +401,6 @@ int ftrace_disable_ftrace_graph_caller(void) | |||
368 | return ftrace_mod_jmp(ip, old_offset, new_offset); | 401 | return ftrace_mod_jmp(ip, old_offset, new_offset); |
369 | } | 402 | } |
370 | 403 | ||
371 | #else /* CONFIG_DYNAMIC_FTRACE */ | ||
372 | |||
373 | /* | ||
374 | * These functions are picked from those used on | ||
375 | * this page for dynamic ftrace. They have been | ||
376 | * simplified to ignore all traces in NMI context. | ||
377 | */ | ||
378 | static atomic_t in_nmi; | ||
379 | |||
380 | void ftrace_nmi_enter(void) | ||
381 | { | ||
382 | atomic_inc(&in_nmi); | ||
383 | } | ||
384 | |||
385 | void ftrace_nmi_exit(void) | ||
386 | { | ||
387 | atomic_dec(&in_nmi); | ||
388 | } | ||
389 | |||
390 | #endif /* !CONFIG_DYNAMIC_FTRACE */ | 404 | #endif /* !CONFIG_DYNAMIC_FTRACE */ |
391 | 405 | ||
392 | /* | 406 | /* |
@@ -396,14 +410,13 @@ void ftrace_nmi_exit(void) | |||
396 | void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) | 410 | void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) |
397 | { | 411 | { |
398 | unsigned long old; | 412 | unsigned long old; |
399 | unsigned long long calltime; | ||
400 | int faulted; | 413 | int faulted; |
401 | struct ftrace_graph_ent trace; | 414 | struct ftrace_graph_ent trace; |
402 | unsigned long return_hooker = (unsigned long) | 415 | unsigned long return_hooker = (unsigned long) |
403 | &return_to_handler; | 416 | &return_to_handler; |
404 | 417 | ||
405 | /* Nmi's are currently unsupported */ | 418 | /* Nmi's are currently unsupported */ |
406 | if (unlikely(atomic_read(&in_nmi))) | 419 | if (unlikely(in_nmi())) |
407 | return; | 420 | return; |
408 | 421 | ||
409 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) | 422 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
@@ -439,17 +452,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) | |||
439 | return; | 452 | return; |
440 | } | 453 | } |
441 | 454 | ||
442 | if (unlikely(!__kernel_text_address(old))) { | 455 | if (ftrace_push_return_trace(old, self_addr, &trace.depth) == -EBUSY) { |
443 | ftrace_graph_stop(); | ||
444 | *parent = old; | ||
445 | WARN_ON(1); | ||
446 | return; | ||
447 | } | ||
448 | |||
449 | calltime = cpu_clock(raw_smp_processor_id()); | ||
450 | |||
451 | if (ftrace_push_return_trace(old, calltime, | ||
452 | self_addr, &trace.depth) == -EBUSY) { | ||
453 | *parent = old; | 456 | *parent = old; |
454 | return; | 457 | return; |
455 | } | 458 | } |
@@ -463,3 +466,66 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) | |||
463 | } | 466 | } |
464 | } | 467 | } |
465 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | 468 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
469 | |||
470 | #ifdef CONFIG_FTRACE_SYSCALLS | ||
471 | |||
472 | extern unsigned long __start_syscalls_metadata[]; | ||
473 | extern unsigned long __stop_syscalls_metadata[]; | ||
474 | extern unsigned long *sys_call_table; | ||
475 | |||
476 | static struct syscall_metadata **syscalls_metadata; | ||
477 | |||
478 | static struct syscall_metadata *find_syscall_meta(unsigned long *syscall) | ||
479 | { | ||
480 | struct syscall_metadata *start; | ||
481 | struct syscall_metadata *stop; | ||
482 | char str[KSYM_SYMBOL_LEN]; | ||
483 | |||
484 | |||
485 | start = (struct syscall_metadata *)__start_syscalls_metadata; | ||
486 | stop = (struct syscall_metadata *)__stop_syscalls_metadata; | ||
487 | kallsyms_lookup((unsigned long) syscall, NULL, NULL, NULL, str); | ||
488 | |||
489 | for ( ; start < stop; start++) { | ||
490 | if (start->name && !strcmp(start->name, str)) | ||
491 | return start; | ||
492 | } | ||
493 | return NULL; | ||
494 | } | ||
495 | |||
496 | struct syscall_metadata *syscall_nr_to_meta(int nr) | ||
497 | { | ||
498 | if (!syscalls_metadata || nr >= FTRACE_SYSCALL_MAX || nr < 0) | ||
499 | return NULL; | ||
500 | |||
501 | return syscalls_metadata[nr]; | ||
502 | } | ||
503 | |||
504 | void arch_init_ftrace_syscalls(void) | ||
505 | { | ||
506 | int i; | ||
507 | struct syscall_metadata *meta; | ||
508 | unsigned long **psys_syscall_table = &sys_call_table; | ||
509 | static atomic_t refs; | ||
510 | |||
511 | if (atomic_inc_return(&refs) != 1) | ||
512 | goto end; | ||
513 | |||
514 | syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) * | ||
515 | FTRACE_SYSCALL_MAX, GFP_KERNEL); | ||
516 | if (!syscalls_metadata) { | ||
517 | WARN_ON(1); | ||
518 | return; | ||
519 | } | ||
520 | |||
521 | for (i = 0; i < FTRACE_SYSCALL_MAX; i++) { | ||
522 | meta = find_syscall_meta(psys_syscall_table[i]); | ||
523 | syscalls_metadata[i] = meta; | ||
524 | } | ||
525 | return; | ||
526 | |||
527 | /* Paranoid: avoid overflow */ | ||
528 | end: | ||
529 | atomic_dec(&refs); | ||
530 | } | ||
531 | #endif | ||
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c index 55b94614e348..7b5169d2b000 100644 --- a/arch/x86/kernel/kprobes.c +++ b/arch/x86/kernel/kprobes.c | |||
@@ -638,13 +638,13 @@ static void __used __kprobes kretprobe_trampoline_holder(void) | |||
638 | #else | 638 | #else |
639 | " pushf\n" | 639 | " pushf\n" |
640 | /* | 640 | /* |
641 | * Skip cs, ip, orig_ax. | 641 | * Skip cs, ip, orig_ax and gs. |
642 | * trampoline_handler() will plug in these values | 642 | * trampoline_handler() will plug in these values |
643 | */ | 643 | */ |
644 | " subl $12, %esp\n" | 644 | " subl $16, %esp\n" |
645 | " pushl %fs\n" | 645 | " pushl %fs\n" |
646 | " pushl %ds\n" | ||
647 | " pushl %es\n" | 646 | " pushl %es\n" |
647 | " pushl %ds\n" | ||
648 | " pushl %eax\n" | 648 | " pushl %eax\n" |
649 | " pushl %ebp\n" | 649 | " pushl %ebp\n" |
650 | " pushl %edi\n" | 650 | " pushl %edi\n" |
@@ -655,10 +655,10 @@ static void __used __kprobes kretprobe_trampoline_holder(void) | |||
655 | " movl %esp, %eax\n" | 655 | " movl %esp, %eax\n" |
656 | " call trampoline_handler\n" | 656 | " call trampoline_handler\n" |
657 | /* Move flags to cs */ | 657 | /* Move flags to cs */ |
658 | " movl 52(%esp), %edx\n" | 658 | " movl 56(%esp), %edx\n" |
659 | " movl %edx, 48(%esp)\n" | 659 | " movl %edx, 52(%esp)\n" |
660 | /* Replace saved flags with true return address. */ | 660 | /* Replace saved flags with true return address. */ |
661 | " movl %eax, 52(%esp)\n" | 661 | " movl %eax, 56(%esp)\n" |
662 | " popl %ebx\n" | 662 | " popl %ebx\n" |
663 | " popl %ecx\n" | 663 | " popl %ecx\n" |
664 | " popl %edx\n" | 664 | " popl %edx\n" |
@@ -666,8 +666,8 @@ static void __used __kprobes kretprobe_trampoline_holder(void) | |||
666 | " popl %edi\n" | 666 | " popl %edi\n" |
667 | " popl %ebp\n" | 667 | " popl %ebp\n" |
668 | " popl %eax\n" | 668 | " popl %eax\n" |
669 | /* Skip ip, orig_ax, es, ds, fs */ | 669 | /* Skip ds, es, fs, gs, orig_ax and ip */ |
670 | " addl $20, %esp\n" | 670 | " addl $24, %esp\n" |
671 | " popf\n" | 671 | " popf\n" |
672 | #endif | 672 | #endif |
673 | " ret\n"); | 673 | " ret\n"); |
@@ -691,6 +691,7 @@ static __used __kprobes void *trampoline_handler(struct pt_regs *regs) | |||
691 | regs->cs = __KERNEL_CS; | 691 | regs->cs = __KERNEL_CS; |
692 | #else | 692 | #else |
693 | regs->cs = __KERNEL_CS | get_kernel_rpl(); | 693 | regs->cs = __KERNEL_CS | get_kernel_rpl(); |
694 | regs->gs = 0; | ||
694 | #endif | 695 | #endif |
695 | regs->ip = trampoline_address; | 696 | regs->ip = trampoline_address; |
696 | regs->orig_ax = ~0UL; | 697 | regs->orig_ax = ~0UL; |
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 156f87582c6c..62fc75b67e45 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c | |||
@@ -8,7 +8,7 @@ | |||
8 | #include <linux/module.h> | 8 | #include <linux/module.h> |
9 | #include <linux/pm.h> | 9 | #include <linux/pm.h> |
10 | #include <linux/clockchips.h> | 10 | #include <linux/clockchips.h> |
11 | #include <linux/ftrace.h> | 11 | #include <trace/power.h> |
12 | #include <asm/system.h> | 12 | #include <asm/system.h> |
13 | #include <asm/apic.h> | 13 | #include <asm/apic.h> |
14 | #include <asm/idle.h> | 14 | #include <asm/idle.h> |
@@ -22,6 +22,9 @@ EXPORT_SYMBOL(idle_nomwait); | |||
22 | 22 | ||
23 | struct kmem_cache *task_xstate_cachep; | 23 | struct kmem_cache *task_xstate_cachep; |
24 | 24 | ||
25 | DEFINE_TRACE(power_start); | ||
26 | DEFINE_TRACE(power_end); | ||
27 | |||
25 | int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) | 28 | int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) |
26 | { | 29 | { |
27 | *dst = *src; | 30 | *dst = *src; |
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index 19378715f415..5c6e46320db1 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c | |||
@@ -21,6 +21,7 @@ | |||
21 | #include <linux/audit.h> | 21 | #include <linux/audit.h> |
22 | #include <linux/seccomp.h> | 22 | #include <linux/seccomp.h> |
23 | #include <linux/signal.h> | 23 | #include <linux/signal.h> |
24 | #include <linux/ftrace.h> | ||
24 | 25 | ||
25 | #include <asm/uaccess.h> | 26 | #include <asm/uaccess.h> |
26 | #include <asm/pgtable.h> | 27 | #include <asm/pgtable.h> |
@@ -1415,6 +1416,9 @@ asmregparm long syscall_trace_enter(struct pt_regs *regs) | |||
1415 | tracehook_report_syscall_entry(regs)) | 1416 | tracehook_report_syscall_entry(regs)) |
1416 | ret = -1L; | 1417 | ret = -1L; |
1417 | 1418 | ||
1419 | if (unlikely(test_thread_flag(TIF_SYSCALL_FTRACE))) | ||
1420 | ftrace_syscall_enter(regs); | ||
1421 | |||
1418 | if (unlikely(current->audit_context)) { | 1422 | if (unlikely(current->audit_context)) { |
1419 | if (IS_IA32) | 1423 | if (IS_IA32) |
1420 | audit_syscall_entry(AUDIT_ARCH_I386, | 1424 | audit_syscall_entry(AUDIT_ARCH_I386, |
@@ -1438,6 +1442,9 @@ asmregparm void syscall_trace_leave(struct pt_regs *regs) | |||
1438 | if (unlikely(current->audit_context)) | 1442 | if (unlikely(current->audit_context)) |
1439 | audit_syscall_exit(AUDITSC_RESULT(regs->ax), regs->ax); | 1443 | audit_syscall_exit(AUDITSC_RESULT(regs->ax), regs->ax); |
1440 | 1444 | ||
1445 | if (unlikely(test_thread_flag(TIF_SYSCALL_FTRACE))) | ||
1446 | ftrace_syscall_exit(regs); | ||
1447 | |||
1441 | if (test_thread_flag(TIF_SYSCALL_TRACE)) | 1448 | if (test_thread_flag(TIF_SYSCALL_TRACE)) |
1442 | tracehook_report_syscall_exit(regs, 0); | 1449 | tracehook_report_syscall_exit(regs, 0); |
1443 | 1450 | ||
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 0a303c3ed11f..a58504ea78cc 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig | |||
@@ -59,7 +59,8 @@ config KVM_AMD | |||
59 | 59 | ||
60 | config KVM_TRACE | 60 | config KVM_TRACE |
61 | bool "KVM trace support" | 61 | bool "KVM trace support" |
62 | depends on KVM && MARKERS && SYSFS | 62 | depends on KVM && SYSFS |
63 | select MARKERS | ||
63 | select RELAY | 64 | select RELAY |
64 | select DEBUG_FS | 65 | select DEBUG_FS |
65 | default n | 66 | default n |
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index db81e9a8556b..749559ed80f5 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c | |||
@@ -1054,17 +1054,47 @@ static noinline int do_test_wp_bit(void) | |||
1054 | const int rodata_test_data = 0xC3; | 1054 | const int rodata_test_data = 0xC3; |
1055 | EXPORT_SYMBOL_GPL(rodata_test_data); | 1055 | EXPORT_SYMBOL_GPL(rodata_test_data); |
1056 | 1056 | ||
1057 | static int kernel_set_to_readonly; | ||
1058 | |||
1059 | void set_kernel_text_rw(void) | ||
1060 | { | ||
1061 | unsigned long start = PFN_ALIGN(_text); | ||
1062 | unsigned long size = PFN_ALIGN(_etext) - start; | ||
1063 | |||
1064 | if (!kernel_set_to_readonly) | ||
1065 | return; | ||
1066 | |||
1067 | pr_debug("Set kernel text: %lx - %lx for read write\n", | ||
1068 | start, start+size); | ||
1069 | |||
1070 | set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT); | ||
1071 | } | ||
1072 | |||
1073 | void set_kernel_text_ro(void) | ||
1074 | { | ||
1075 | unsigned long start = PFN_ALIGN(_text); | ||
1076 | unsigned long size = PFN_ALIGN(_etext) - start; | ||
1077 | |||
1078 | if (!kernel_set_to_readonly) | ||
1079 | return; | ||
1080 | |||
1081 | pr_debug("Set kernel text: %lx - %lx for read only\n", | ||
1082 | start, start+size); | ||
1083 | |||
1084 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); | ||
1085 | } | ||
1086 | |||
1057 | void mark_rodata_ro(void) | 1087 | void mark_rodata_ro(void) |
1058 | { | 1088 | { |
1059 | unsigned long start = PFN_ALIGN(_text); | 1089 | unsigned long start = PFN_ALIGN(_text); |
1060 | unsigned long size = PFN_ALIGN(_etext) - start; | 1090 | unsigned long size = PFN_ALIGN(_etext) - start; |
1061 | 1091 | ||
1062 | #ifndef CONFIG_DYNAMIC_FTRACE | ||
1063 | /* Dynamic tracing modifies the kernel text section */ | ||
1064 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); | 1092 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
1065 | printk(KERN_INFO "Write protecting the kernel text: %luk\n", | 1093 | printk(KERN_INFO "Write protecting the kernel text: %luk\n", |
1066 | size >> 10); | 1094 | size >> 10); |
1067 | 1095 | ||
1096 | kernel_set_to_readonly = 1; | ||
1097 | |||
1068 | #ifdef CONFIG_CPA_DEBUG | 1098 | #ifdef CONFIG_CPA_DEBUG |
1069 | printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n", | 1099 | printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n", |
1070 | start, start+size); | 1100 | start, start+size); |
@@ -1073,7 +1103,6 @@ void mark_rodata_ro(void) | |||
1073 | printk(KERN_INFO "Testing CPA: write protecting again\n"); | 1103 | printk(KERN_INFO "Testing CPA: write protecting again\n"); |
1074 | set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT); | 1104 | set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT); |
1075 | #endif | 1105 | #endif |
1076 | #endif /* CONFIG_DYNAMIC_FTRACE */ | ||
1077 | 1106 | ||
1078 | start += size; | 1107 | start += size; |
1079 | size = (unsigned long)__end_rodata - start; | 1108 | size = (unsigned long)__end_rodata - start; |
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 54efa57d1c03..1753e8020df6 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c | |||
@@ -734,21 +734,48 @@ void __init mem_init(void) | |||
734 | const int rodata_test_data = 0xC3; | 734 | const int rodata_test_data = 0xC3; |
735 | EXPORT_SYMBOL_GPL(rodata_test_data); | 735 | EXPORT_SYMBOL_GPL(rodata_test_data); |
736 | 736 | ||
737 | static int kernel_set_to_readonly; | ||
738 | |||
739 | void set_kernel_text_rw(void) | ||
740 | { | ||
741 | unsigned long start = PFN_ALIGN(_stext); | ||
742 | unsigned long end = PFN_ALIGN(__start_rodata); | ||
743 | |||
744 | if (!kernel_set_to_readonly) | ||
745 | return; | ||
746 | |||
747 | pr_debug("Set kernel text: %lx - %lx for read write\n", | ||
748 | start, end); | ||
749 | |||
750 | set_memory_rw(start, (end - start) >> PAGE_SHIFT); | ||
751 | } | ||
752 | |||
753 | void set_kernel_text_ro(void) | ||
754 | { | ||
755 | unsigned long start = PFN_ALIGN(_stext); | ||
756 | unsigned long end = PFN_ALIGN(__start_rodata); | ||
757 | |||
758 | if (!kernel_set_to_readonly) | ||
759 | return; | ||
760 | |||
761 | pr_debug("Set kernel text: %lx - %lx for read only\n", | ||
762 | start, end); | ||
763 | |||
764 | set_memory_ro(start, (end - start) >> PAGE_SHIFT); | ||
765 | } | ||
766 | |||
737 | void mark_rodata_ro(void) | 767 | void mark_rodata_ro(void) |
738 | { | 768 | { |
739 | unsigned long start = PFN_ALIGN(_stext), end = PFN_ALIGN(__end_rodata); | 769 | unsigned long start = PFN_ALIGN(_stext), end = PFN_ALIGN(__end_rodata); |
740 | unsigned long rodata_start = | 770 | unsigned long rodata_start = |
741 | ((unsigned long)__start_rodata + PAGE_SIZE - 1) & PAGE_MASK; | 771 | ((unsigned long)__start_rodata + PAGE_SIZE - 1) & PAGE_MASK; |
742 | 772 | ||
743 | #ifdef CONFIG_DYNAMIC_FTRACE | ||
744 | /* Dynamic tracing modifies the kernel text section */ | ||
745 | start = rodata_start; | ||
746 | #endif | ||
747 | |||
748 | printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", | 773 | printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", |
749 | (end - start) >> 10); | 774 | (end - start) >> 10); |
750 | set_memory_ro(start, (end - start) >> PAGE_SHIFT); | 775 | set_memory_ro(start, (end - start) >> PAGE_SHIFT); |
751 | 776 | ||
777 | kernel_set_to_readonly = 1; | ||
778 | |||
752 | /* | 779 | /* |
753 | * The rodata section (but not the kernel text!) should also be | 780 | * The rodata section (but not the kernel text!) should also be |
754 | * not-executable. | 781 | * not-executable. |
diff --git a/arch/xtensa/include/asm/ftrace.h b/arch/xtensa/include/asm/ftrace.h new file mode 100644 index 000000000000..40a8c178f10d --- /dev/null +++ b/arch/xtensa/include/asm/ftrace.h | |||
@@ -0,0 +1 @@ | |||
/* empty */ | |||