aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/ABI/testing/debugfs-kmemtrace71
-rw-r--r--Documentation/kernel-parameters.txt10
-rw-r--r--Documentation/sysrq.txt2
-rw-r--r--Documentation/tracers/mmiotrace.txt6
-rw-r--r--Documentation/vm/kmemtrace.txt126
-rw-r--r--MAINTAINERS6
-rw-r--r--arch/ia64/Kconfig3
-rw-r--r--arch/ia64/include/asm/ftrace.h28
-rw-r--r--arch/ia64/kernel/Makefile5
-rw-r--r--arch/ia64/kernel/entry.S100
-rw-r--r--arch/ia64/kernel/ftrace.c206
-rw-r--r--arch/ia64/kernel/ia64_ksyms.c6
-rw-r--r--arch/x86/Kconfig.debug24
-rw-r--r--arch/x86/kvm/Kconfig3
-rw-r--r--drivers/char/sysrq.c2
-rw-r--r--include/linux/ftrace.h4
-rw-r--r--include/linux/slab_def.h68
-rw-r--r--include/linux/slob_def.h9
-rw-r--r--include/linux/slub_def.h53
-rw-r--r--include/trace/kmemtrace.h75
-rw-r--r--include/trace/workqueue.h25
-rw-r--r--init/main.c2
-rw-r--r--kernel/relay.c4
-rw-r--r--kernel/trace/Kconfig55
-rw-r--r--kernel/trace/Makefile4
-rw-r--r--kernel/trace/ftrace.c45
-rw-r--r--kernel/trace/kmemtrace.c350
-rw-r--r--kernel/trace/ring_buffer.c50
-rw-r--r--kernel/trace/trace.c857
-rw-r--r--kernel/trace/trace.h59
-rw-r--r--kernel/trace/trace_boot.c1
-rw-r--r--kernel/trace/trace_branch.c278
-rw-r--r--kernel/trace/trace_functions.c193
-rw-r--r--kernel/trace/trace_functions_graph.c8
-rw-r--r--kernel/trace/trace_hw_branches.c1
-rw-r--r--kernel/trace/trace_irqsoff.c1
-rw-r--r--kernel/trace/trace_mmiotrace.c31
-rw-r--r--kernel/trace/trace_output.c829
-rw-r--r--kernel/trace/trace_output.h60
-rw-r--r--kernel/trace/trace_power.c1
-rw-r--r--kernel/trace/trace_sched_wakeup.c1
-rw-r--r--kernel/trace/trace_selftest.c1
-rw-r--r--kernel/trace/trace_stat.c319
-rw-r--r--kernel/trace/trace_stat.h31
-rw-r--r--kernel/trace/trace_workqueue.c287
-rw-r--r--kernel/workqueue.c16
-rw-r--r--mm/slab.c71
-rw-r--r--mm/slob.c37
-rw-r--r--mm/slub.c83
-rw-r--r--scripts/Makefile.build13
-rwxr-xr-xscripts/recordmcount.pl18
51 files changed, 3470 insertions, 1068 deletions
diff --git a/Documentation/ABI/testing/debugfs-kmemtrace b/Documentation/ABI/testing/debugfs-kmemtrace
new file mode 100644
index 000000000000..5e6a92a02d85
--- /dev/null
+++ b/Documentation/ABI/testing/debugfs-kmemtrace
@@ -0,0 +1,71 @@
1What: /sys/kernel/debug/kmemtrace/
2Date: July 2008
3Contact: Eduard - Gabriel Munteanu <eduard.munteanu@linux360.ro>
4Description:
5
6In kmemtrace-enabled kernels, the following files are created:
7
8/sys/kernel/debug/kmemtrace/
9 cpu<n> (0400) Per-CPU tracing data, see below. (binary)
10 total_overruns (0400) Total number of bytes which were dropped from
11 cpu<n> files because of full buffer condition,
12 non-binary. (text)
13 abi_version (0400) Kernel's kmemtrace ABI version. (text)
14
15Each per-CPU file should be read according to the relay interface. That is,
16the reader should set affinity to that specific CPU and, as currently done by
17the userspace application (though there are other methods), use poll() with
18an infinite timeout before every read(). Otherwise, erroneous data may be
19read. The binary data has the following _core_ format:
20
21 Event ID (1 byte) Unsigned integer, one of:
22 0 - represents an allocation (KMEMTRACE_EVENT_ALLOC)
23 1 - represents a freeing of previously allocated memory
24 (KMEMTRACE_EVENT_FREE)
25 Type ID (1 byte) Unsigned integer, one of:
26 0 - this is a kmalloc() / kfree()
27 1 - this is a kmem_cache_alloc() / kmem_cache_free()
28 2 - this is a __get_free_pages() et al.
29 Event size (2 bytes) Unsigned integer representing the
30 size of this event. Used to extend
31 kmemtrace. Discard the bytes you
32 don't know about.
33 Sequence number (4 bytes) Signed integer used to reorder data
34 logged on SMP machines. Wraparound
35 must be taken into account, although
36 it is unlikely.
37 Caller address (8 bytes) Return address to the caller.
38 Pointer to mem (8 bytes) Pointer to target memory area. Can be
39 NULL, but not all such calls might be
40 recorded.
41
42In case of KMEMTRACE_EVENT_ALLOC events, the next fields follow:
43
44 Requested bytes (8 bytes) Total number of requested bytes,
45 unsigned, must not be zero.
46 Allocated bytes (8 bytes) Total number of actually allocated
47 bytes, unsigned, must not be lower
48 than requested bytes.
49 Requested flags (4 bytes) GFP flags supplied by the caller.
50 Target CPU (4 bytes) Signed integer, valid for event id 1.
51 If equal to -1, target CPU is the same
52 as origin CPU, but the reverse might
53 not be true.
54
55The data is made available in the same endianness the machine has.
56
57Other event ids and type ids may be defined and added. Other fields may be
58added by increasing event size, but see below for details.
59Every modification to the ABI, including new id definitions, are followed
60by bumping the ABI version by one.
61
62Adding new data to the packet (features) is done at the end of the mandatory
63data:
64 Feature size (2 byte)
65 Feature ID (1 byte)
66 Feature data (Feature size - 3 bytes)
67
68
69Users:
70 kmemtrace-user - git://repo.or.cz/kmemtrace-user.git
71
diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index 8511d3532c27..ac613a661f46 100644
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -49,6 +49,7 @@ parameter is applicable:
49 ISAPNP ISA PnP code is enabled. 49 ISAPNP ISA PnP code is enabled.
50 ISDN Appropriate ISDN support is enabled. 50 ISDN Appropriate ISDN support is enabled.
51 JOY Appropriate joystick support is enabled. 51 JOY Appropriate joystick support is enabled.
52 KMEMTRACE kmemtrace is enabled.
52 LIBATA Libata driver is enabled 53 LIBATA Libata driver is enabled
53 LP Printer support is enabled. 54 LP Printer support is enabled.
54 LOOP Loopback device support is enabled. 55 LOOP Loopback device support is enabled.
@@ -1050,6 +1051,15 @@ and is between 256 and 4096 characters. It is defined in the file
1050 use the HighMem zone if it exists, and the Normal 1051 use the HighMem zone if it exists, and the Normal
1051 zone if it does not. 1052 zone if it does not.
1052 1053
1054 kmemtrace.enable= [KNL,KMEMTRACE] Format: { yes | no }
1055 Controls whether kmemtrace is enabled
1056 at boot-time.
1057
1058 kmemtrace.subbufs=n [KNL,KMEMTRACE] Overrides the number of
1059 subbufs kmemtrace's relay channel has. Set this
1060 higher than default (KMEMTRACE_N_SUBBUFS in code) if
1061 you experience buffer overruns.
1062
1053 movablecore=nn[KMG] [KNL,X86-32,IA-64,PPC,X86-64] This parameter 1063 movablecore=nn[KMG] [KNL,X86-32,IA-64,PPC,X86-64] This parameter
1054 is similar to kernelcore except it specifies the 1064 is similar to kernelcore except it specifies the
1055 amount of memory used for migratable allocations. 1065 amount of memory used for migratable allocations.
diff --git a/Documentation/sysrq.txt b/Documentation/sysrq.txt
index 9e592c718afb..535aeb936dbc 100644
--- a/Documentation/sysrq.txt
+++ b/Documentation/sysrq.txt
@@ -113,6 +113,8 @@ On all - write a character to /proc/sysrq-trigger. e.g.:
113 113
114'x' - Used by xmon interface on ppc/powerpc platforms. 114'x' - Used by xmon interface on ppc/powerpc platforms.
115 115
116'z' - Dump the ftrace buffer
117
116'0'-'9' - Sets the console log level, controlling which kernel messages 118'0'-'9' - Sets the console log level, controlling which kernel messages
117 will be printed to your console. ('0', for example would make 119 will be printed to your console. ('0', for example would make
118 it so that only emergency messages like PANICs or OOPSes would 120 it so that only emergency messages like PANICs or OOPSes would
diff --git a/Documentation/tracers/mmiotrace.txt b/Documentation/tracers/mmiotrace.txt
index cde23b4a12a1..5731c67abc55 100644
--- a/Documentation/tracers/mmiotrace.txt
+++ b/Documentation/tracers/mmiotrace.txt
@@ -78,12 +78,10 @@ to view your kernel log and look for "mmiotrace has lost events" warning. If
78events were lost, the trace is incomplete. You should enlarge the buffers and 78events were lost, the trace is incomplete. You should enlarge the buffers and
79try again. Buffers are enlarged by first seeing how large the current buffers 79try again. Buffers are enlarged by first seeing how large the current buffers
80are: 80are:
81$ cat /debug/tracing/trace_entries 81$ cat /debug/tracing/buffer_size_kb
82gives you a number. Approximately double this number and write it back, for 82gives you a number. Approximately double this number and write it back, for
83instance: 83instance:
84$ echo 0 > /debug/tracing/tracing_enabled 84$ echo 128000 > /debug/tracing/buffer_size_kb
85$ echo 128000 > /debug/tracing/trace_entries
86$ echo 1 > /debug/tracing/tracing_enabled
87Then start again from the top. 85Then start again from the top.
88 86
89If you are doing a trace for a driver project, e.g. Nouveau, you should also 87If you are doing a trace for a driver project, e.g. Nouveau, you should also
diff --git a/Documentation/vm/kmemtrace.txt b/Documentation/vm/kmemtrace.txt
new file mode 100644
index 000000000000..a956d9b7f943
--- /dev/null
+++ b/Documentation/vm/kmemtrace.txt
@@ -0,0 +1,126 @@
1 kmemtrace - Kernel Memory Tracer
2
3 by Eduard - Gabriel Munteanu
4 <eduard.munteanu@linux360.ro>
5
6I. Introduction
7===============
8
9kmemtrace helps kernel developers figure out two things:
101) how different allocators (SLAB, SLUB etc.) perform
112) how kernel code allocates memory and how much
12
13To do this, we trace every allocation and export information to the userspace
14through the relay interface. We export things such as the number of requested
15bytes, the number of bytes actually allocated (i.e. including internal
16fragmentation), whether this is a slab allocation or a plain kmalloc() and so
17on.
18
19The actual analysis is performed by a userspace tool (see section III for
20details on where to get it from). It logs the data exported by the kernel,
21processes it and (as of writing this) can provide the following information:
22- the total amount of memory allocated and fragmentation per call-site
23- the amount of memory allocated and fragmentation per allocation
24- total memory allocated and fragmentation in the collected dataset
25- number of cross-CPU allocation and frees (makes sense in NUMA environments)
26
27Moreover, it can potentially find inconsistent and erroneous behavior in
28kernel code, such as using slab free functions on kmalloc'ed memory or
29allocating less memory than requested (but not truly failed allocations).
30
31kmemtrace also makes provisions for tracing on some arch and analysing the
32data on another.
33
34II. Design and goals
35====================
36
37kmemtrace was designed to handle rather large amounts of data. Thus, it uses
38the relay interface to export whatever is logged to userspace, which then
39stores it. Analysis and reporting is done asynchronously, that is, after the
40data is collected and stored. By design, it allows one to log and analyse
41on different machines and different arches.
42
43As of writing this, the ABI is not considered stable, though it might not
44change much. However, no guarantees are made about compatibility yet. When
45deemed stable, the ABI should still allow easy extension while maintaining
46backward compatibility. This is described further in Documentation/ABI.
47
48Summary of design goals:
49 - allow logging and analysis to be done across different machines
50 - be fast and anticipate usage in high-load environments (*)
51 - be reasonably extensible
52 - make it possible for GNU/Linux distributions to have kmemtrace
53 included in their repositories
54
55(*) - one of the reasons Pekka Enberg's original userspace data analysis
56 tool's code was rewritten from Perl to C (although this is more than a
57 simple conversion)
58
59
60III. Quick usage guide
61======================
62
631) Get a kernel that supports kmemtrace and build it accordingly (i.e. enable
64CONFIG_KMEMTRACE).
65
662) Get the userspace tool and build it:
67$ git-clone git://repo.or.cz/kmemtrace-user.git # current repository
68$ cd kmemtrace-user/
69$ ./autogen.sh
70$ ./configure
71$ make
72
733) Boot the kmemtrace-enabled kernel if you haven't, preferably in the
74'single' runlevel (so that relay buffers don't fill up easily), and run
75kmemtrace:
76# '$' does not mean user, but root here.
77$ mount -t debugfs none /sys/kernel/debug
78$ mount -t proc none /proc
79$ cd path/to/kmemtrace-user/
80$ ./kmemtraced
81Wait a bit, then stop it with CTRL+C.
82$ cat /sys/kernel/debug/kmemtrace/total_overruns # Check if we didn't
83 # overrun, should
84 # be zero.
85$ (Optionally) [Run kmemtrace_check separately on each cpu[0-9]*.out file to
86 check its correctness]
87$ ./kmemtrace-report
88
89Now you should have a nice and short summary of how the allocator performs.
90
91IV. FAQ and known issues
92========================
93
94Q: 'cat /sys/kernel/debug/kmemtrace/total_overruns' is non-zero, how do I fix
95this? Should I worry?
96A: If it's non-zero, this affects kmemtrace's accuracy, depending on how
97large the number is. You can fix it by supplying a higher
98'kmemtrace.subbufs=N' kernel parameter.
99---
100
101Q: kmemtrace_check reports errors, how do I fix this? Should I worry?
102A: This is a bug and should be reported. It can occur for a variety of
103reasons:
104 - possible bugs in relay code
105 - possible misuse of relay by kmemtrace
106 - timestamps being collected unorderly
107Or you may fix it yourself and send us a patch.
108---
109
110Q: kmemtrace_report shows many errors, how do I fix this? Should I worry?
111A: This is a known issue and I'm working on it. These might be true errors
112in kernel code, which may have inconsistent behavior (e.g. allocating memory
113with kmem_cache_alloc() and freeing it with kfree()). Pekka Enberg pointed
114out this behavior may work with SLAB, but may fail with other allocators.
115
116It may also be due to lack of tracing in some unusual allocator functions.
117
118We don't want bug reports regarding this issue yet.
119---
120
121V. See also
122===========
123
124Documentation/kernel-parameters.txt
125Documentation/ABI/testing/debugfs-kmemtrace
126
diff --git a/MAINTAINERS b/MAINTAINERS
index 3fe4dc2c2564..c3c52779fa6c 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2605,6 +2605,12 @@ M: jason.wessel@windriver.com
2605L: kgdb-bugreport@lists.sourceforge.net 2605L: kgdb-bugreport@lists.sourceforge.net
2606S: Maintained 2606S: Maintained
2607 2607
2608KMEMTRACE
2609P: Eduard - Gabriel Munteanu
2610M: eduard.munteanu@linux360.ro
2611L: linux-kernel@vger.kernel.org
2612S: Maintained
2613
2608KPROBES 2614KPROBES
2609P: Ananth N Mavinakayanahalli 2615P: Ananth N Mavinakayanahalli
2610M: ananth@in.ibm.com 2616M: ananth@in.ibm.com
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 6183aeccecf1..8b6a8a554afa 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -22,6 +22,9 @@ config IA64
22 select HAVE_OPROFILE 22 select HAVE_OPROFILE
23 select HAVE_KPROBES 23 select HAVE_KPROBES
24 select HAVE_KRETPROBES 24 select HAVE_KRETPROBES
25 select HAVE_FTRACE_MCOUNT_RECORD
26 select HAVE_DYNAMIC_FTRACE if (!ITANIUM)
27 select HAVE_FUNCTION_TRACER
25 select HAVE_DMA_ATTRS 28 select HAVE_DMA_ATTRS
26 select HAVE_KVM 29 select HAVE_KVM
27 select HAVE_ARCH_TRACEHOOK 30 select HAVE_ARCH_TRACEHOOK
diff --git a/arch/ia64/include/asm/ftrace.h b/arch/ia64/include/asm/ftrace.h
new file mode 100644
index 000000000000..d20db3c2a656
--- /dev/null
+++ b/arch/ia64/include/asm/ftrace.h
@@ -0,0 +1,28 @@
1#ifndef _ASM_IA64_FTRACE_H
2#define _ASM_IA64_FTRACE_H
3
4#ifdef CONFIG_FUNCTION_TRACER
5#define MCOUNT_INSN_SIZE 32 /* sizeof mcount call */
6
7#ifndef __ASSEMBLY__
8extern void _mcount(unsigned long pfs, unsigned long r1, unsigned long b0, unsigned long r0);
9#define mcount _mcount
10
11#include <asm/kprobes.h>
12/* In IA64, MCOUNT_ADDR is set in link time, so it's not a constant at compile time */
13#define MCOUNT_ADDR (((struct fnptr *)mcount)->ip)
14#define FTRACE_ADDR (((struct fnptr *)ftrace_caller)->ip)
15
16static inline unsigned long ftrace_call_adjust(unsigned long addr)
17{
18 /* second bundle, insn 2 */
19 return addr - 0x12;
20}
21
22struct dyn_arch_ftrace {
23};
24#endif
25
26#endif /* CONFIG_FUNCTION_TRACER */
27
28#endif /* _ASM_IA64_FTRACE_H */
diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile
index c381ea954892..ab6e7ec0bba3 100644
--- a/arch/ia64/kernel/Makefile
+++ b/arch/ia64/kernel/Makefile
@@ -2,6 +2,10 @@
2# Makefile for the linux kernel. 2# Makefile for the linux kernel.
3# 3#
4 4
5ifdef CONFIG_DYNAMIC_FTRACE
6CFLAGS_REMOVE_ftrace.o = -pg
7endif
8
5extra-y := head.o init_task.o vmlinux.lds 9extra-y := head.o init_task.o vmlinux.lds
6 10
7obj-y := acpi.o entry.o efi.o efi_stub.o gate-data.o fsys.o ia64_ksyms.o irq.o irq_ia64.o \ 11obj-y := acpi.o entry.o efi.o efi_stub.o gate-data.o fsys.o ia64_ksyms.o irq.o irq_ia64.o \
@@ -28,6 +32,7 @@ obj-$(CONFIG_IA64_CYCLONE) += cyclone.o
28obj-$(CONFIG_CPU_FREQ) += cpufreq/ 32obj-$(CONFIG_CPU_FREQ) += cpufreq/
29obj-$(CONFIG_IA64_MCA_RECOVERY) += mca_recovery.o 33obj-$(CONFIG_IA64_MCA_RECOVERY) += mca_recovery.o
30obj-$(CONFIG_KPROBES) += kprobes.o jprobes.o 34obj-$(CONFIG_KPROBES) += kprobes.o jprobes.o
35obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o
31obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o crash.o 36obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o crash.o
32obj-$(CONFIG_CRASH_DUMP) += crash_dump.o 37obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
33obj-$(CONFIG_IA64_UNCACHED_ALLOCATOR) += uncached.o 38obj-$(CONFIG_IA64_UNCACHED_ALLOCATOR) += uncached.o
diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S
index e5341e2c1175..7e3382b06d56 100644
--- a/arch/ia64/kernel/entry.S
+++ b/arch/ia64/kernel/entry.S
@@ -47,6 +47,7 @@
47#include <asm/processor.h> 47#include <asm/processor.h>
48#include <asm/thread_info.h> 48#include <asm/thread_info.h>
49#include <asm/unistd.h> 49#include <asm/unistd.h>
50#include <asm/ftrace.h>
50 51
51#include "minstate.h" 52#include "minstate.h"
52 53
@@ -1404,6 +1405,105 @@ GLOBAL_ENTRY(unw_init_running)
1404 br.ret.sptk.many rp 1405 br.ret.sptk.many rp
1405END(unw_init_running) 1406END(unw_init_running)
1406 1407
1408#ifdef CONFIG_FUNCTION_TRACER
1409#ifdef CONFIG_DYNAMIC_FTRACE
1410GLOBAL_ENTRY(_mcount)
1411 br ftrace_stub
1412END(_mcount)
1413
1414.here:
1415 br.ret.sptk.many b0
1416
1417GLOBAL_ENTRY(ftrace_caller)
1418 alloc out0 = ar.pfs, 8, 0, 4, 0
1419 mov out3 = r0
1420 ;;
1421 mov out2 = b0
1422 add r3 = 0x20, r3
1423 mov out1 = r1;
1424 br.call.sptk.many b0 = ftrace_patch_gp
1425 //this might be called from module, so we must patch gp
1426ftrace_patch_gp:
1427 movl gp=__gp
1428 mov b0 = r3
1429 ;;
1430.global ftrace_call;
1431ftrace_call:
1432{
1433 .mlx
1434 nop.m 0x0
1435 movl r3 = .here;;
1436}
1437 alloc loc0 = ar.pfs, 4, 4, 2, 0
1438 ;;
1439 mov loc1 = b0
1440 mov out0 = b0
1441 mov loc2 = r8
1442 mov loc3 = r15
1443 ;;
1444 adds out0 = -MCOUNT_INSN_SIZE, out0
1445 mov out1 = in2
1446 mov b6 = r3
1447
1448 br.call.sptk.many b0 = b6
1449 ;;
1450 mov ar.pfs = loc0
1451 mov b0 = loc1
1452 mov r8 = loc2
1453 mov r15 = loc3
1454 br ftrace_stub
1455 ;;
1456END(ftrace_caller)
1457
1458#else
1459GLOBAL_ENTRY(_mcount)
1460 movl r2 = ftrace_stub
1461 movl r3 = ftrace_trace_function;;
1462 ld8 r3 = [r3];;
1463 ld8 r3 = [r3];;
1464 cmp.eq p7,p0 = r2, r3
1465(p7) br.sptk.many ftrace_stub
1466 ;;
1467
1468 alloc loc0 = ar.pfs, 4, 4, 2, 0
1469 ;;
1470 mov loc1 = b0
1471 mov out0 = b0
1472 mov loc2 = r8
1473 mov loc3 = r15
1474 ;;
1475 adds out0 = -MCOUNT_INSN_SIZE, out0
1476 mov out1 = in2
1477 mov b6 = r3
1478
1479 br.call.sptk.many b0 = b6
1480 ;;
1481 mov ar.pfs = loc0
1482 mov b0 = loc1
1483 mov r8 = loc2
1484 mov r15 = loc3
1485 br ftrace_stub
1486 ;;
1487END(_mcount)
1488#endif
1489
1490GLOBAL_ENTRY(ftrace_stub)
1491 mov r3 = b0
1492 movl r2 = _mcount_ret_helper
1493 ;;
1494 mov b6 = r2
1495 mov b7 = r3
1496 br.ret.sptk.many b6
1497
1498_mcount_ret_helper:
1499 mov b0 = r42
1500 mov r1 = r41
1501 mov ar.pfs = r40
1502 br b7
1503END(ftrace_stub)
1504
1505#endif /* CONFIG_FUNCTION_TRACER */
1506
1407 .rodata 1507 .rodata
1408 .align 8 1508 .align 8
1409 .globl sys_call_table 1509 .globl sys_call_table
diff --git a/arch/ia64/kernel/ftrace.c b/arch/ia64/kernel/ftrace.c
new file mode 100644
index 000000000000..7fc8c961b1f7
--- /dev/null
+++ b/arch/ia64/kernel/ftrace.c
@@ -0,0 +1,206 @@
1/*
2 * Dynamic function tracing support.
3 *
4 * Copyright (C) 2008 Shaohua Li <shaohua.li@intel.com>
5 *
6 * For licencing details, see COPYING.
7 *
8 * Defines low-level handling of mcount calls when the kernel
9 * is compiled with the -pg flag. When using dynamic ftrace, the
10 * mcount call-sites get patched lazily with NOP till they are
11 * enabled. All code mutation routines here take effect atomically.
12 */
13
14#include <linux/uaccess.h>
15#include <linux/ftrace.h>
16
17#include <asm/cacheflush.h>
18#include <asm/patch.h>
19
20/* In IA64, each function will be added below two bundles with -pg option */
21static unsigned char __attribute__((aligned(8)))
22ftrace_orig_code[MCOUNT_INSN_SIZE] = {
23 0x02, 0x40, 0x31, 0x10, 0x80, 0x05, /* alloc r40=ar.pfs,12,8,0 */
24 0xb0, 0x02, 0x00, 0x00, 0x42, 0x40, /* mov r43=r0;; */
25 0x05, 0x00, 0xc4, 0x00, /* mov r42=b0 */
26 0x11, 0x48, 0x01, 0x02, 0x00, 0x21, /* mov r41=r1 */
27 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /* nop.i 0x0 */
28 0x08, 0x00, 0x00, 0x50 /* br.call.sptk.many b0 = _mcount;; */
29};
30
31struct ftrace_orig_insn {
32 u64 dummy1, dummy2, dummy3;
33 u64 dummy4:64-41+13;
34 u64 imm20:20;
35 u64 dummy5:3;
36 u64 sign:1;
37 u64 dummy6:4;
38};
39
40/* mcount stub will be converted below for nop */
41static unsigned char ftrace_nop_code[MCOUNT_INSN_SIZE] = {
42 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MII] nop.m 0x0 */
43 0x30, 0x00, 0x00, 0x60, 0x00, 0x00, /* mov r3=ip */
44 0x00, 0x00, 0x04, 0x00, /* nop.i 0x0 */
45 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0x0 */
46 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* nop.x 0x0;; */
47 0x00, 0x00, 0x04, 0x00
48};
49
50static unsigned char *ftrace_nop_replace(void)
51{
52 return ftrace_nop_code;
53}
54
55/*
56 * mcount stub will be converted below for call
57 * Note: Just the last instruction is changed against nop
58 * */
59static unsigned char __attribute__((aligned(8)))
60ftrace_call_code[MCOUNT_INSN_SIZE] = {
61 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MII] nop.m 0x0 */
62 0x30, 0x00, 0x00, 0x60, 0x00, 0x00, /* mov r3=ip */
63 0x00, 0x00, 0x04, 0x00, /* nop.i 0x0 */
64 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0x0 */
65 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, /* brl.many .;;*/
66 0xf8, 0xff, 0xff, 0xc8
67};
68
69struct ftrace_call_insn {
70 u64 dummy1, dummy2;
71 u64 dummy3:48;
72 u64 imm39_l:16;
73 u64 imm39_h:23;
74 u64 dummy4:13;
75 u64 imm20:20;
76 u64 dummy5:3;
77 u64 i:1;
78 u64 dummy6:4;
79};
80
81static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
82{
83 struct ftrace_call_insn *code = (void *)ftrace_call_code;
84 unsigned long offset = addr - (ip + 0x10);
85
86 code->imm39_l = offset >> 24;
87 code->imm39_h = offset >> 40;
88 code->imm20 = offset >> 4;
89 code->i = offset >> 63;
90 return ftrace_call_code;
91}
92
93static int
94ftrace_modify_code(unsigned long ip, unsigned char *old_code,
95 unsigned char *new_code, int do_check)
96{
97 unsigned char replaced[MCOUNT_INSN_SIZE];
98
99 /*
100 * Note: Due to modules and __init, code can
101 * disappear and change, we need to protect against faulting
102 * as well as code changing. We do this by using the
103 * probe_kernel_* functions.
104 *
105 * No real locking needed, this code is run through
106 * kstop_machine, or before SMP starts.
107 */
108
109 if (!do_check)
110 goto skip_check;
111
112 /* read the text we want to modify */
113 if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
114 return -EFAULT;
115
116 /* Make sure it is what we expect it to be */
117 if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
118 return -EINVAL;
119
120skip_check:
121 /* replace the text with the new text */
122 if (probe_kernel_write(((void *)ip), new_code, MCOUNT_INSN_SIZE))
123 return -EPERM;
124 flush_icache_range(ip, ip + MCOUNT_INSN_SIZE);
125
126 return 0;
127}
128
129static int ftrace_make_nop_check(struct dyn_ftrace *rec, unsigned long addr)
130{
131 unsigned char __attribute__((aligned(8))) replaced[MCOUNT_INSN_SIZE];
132 unsigned long ip = rec->ip;
133
134 if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
135 return -EFAULT;
136 if (rec->flags & FTRACE_FL_CONVERTED) {
137 struct ftrace_call_insn *call_insn, *tmp_call;
138
139 call_insn = (void *)ftrace_call_code;
140 tmp_call = (void *)replaced;
141 call_insn->imm39_l = tmp_call->imm39_l;
142 call_insn->imm39_h = tmp_call->imm39_h;
143 call_insn->imm20 = tmp_call->imm20;
144 call_insn->i = tmp_call->i;
145 if (memcmp(replaced, ftrace_call_code, MCOUNT_INSN_SIZE) != 0)
146 return -EINVAL;
147 return 0;
148 } else {
149 struct ftrace_orig_insn *call_insn, *tmp_call;
150
151 call_insn = (void *)ftrace_orig_code;
152 tmp_call = (void *)replaced;
153 call_insn->sign = tmp_call->sign;
154 call_insn->imm20 = tmp_call->imm20;
155 if (memcmp(replaced, ftrace_orig_code, MCOUNT_INSN_SIZE) != 0)
156 return -EINVAL;
157 return 0;
158 }
159}
160
161int ftrace_make_nop(struct module *mod,
162 struct dyn_ftrace *rec, unsigned long addr)
163{
164 int ret;
165 char *new;
166
167 ret = ftrace_make_nop_check(rec, addr);
168 if (ret)
169 return ret;
170 new = ftrace_nop_replace();
171 return ftrace_modify_code(rec->ip, NULL, new, 0);
172}
173
174int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
175{
176 unsigned long ip = rec->ip;
177 unsigned char *old, *new;
178
179 old= ftrace_nop_replace();
180 new = ftrace_call_replace(ip, addr);
181 return ftrace_modify_code(ip, old, new, 1);
182}
183
184/* in IA64, _mcount can't directly call ftrace_stub. Only jump is ok */
185int ftrace_update_ftrace_func(ftrace_func_t func)
186{
187 unsigned long ip;
188 unsigned long addr = ((struct fnptr *)ftrace_call)->ip;
189
190 if (func == ftrace_stub)
191 return 0;
192 ip = ((struct fnptr *)func)->ip;
193
194 ia64_patch_imm64(addr + 2, ip);
195
196 flush_icache_range(addr, addr + 16);
197 return 0;
198}
199
200/* run from kstop_machine */
201int __init ftrace_dyn_arch_init(void *data)
202{
203 *(unsigned long *)data = 0;
204
205 return 0;
206}
diff --git a/arch/ia64/kernel/ia64_ksyms.c b/arch/ia64/kernel/ia64_ksyms.c
index 6da1f20d7372..2d311864e359 100644
--- a/arch/ia64/kernel/ia64_ksyms.c
+++ b/arch/ia64/kernel/ia64_ksyms.c
@@ -112,3 +112,9 @@ EXPORT_SYMBOL_GPL(esi_call_phys);
112#endif 112#endif
113extern char ia64_ivt[]; 113extern char ia64_ivt[];
114EXPORT_SYMBOL(ia64_ivt); 114EXPORT_SYMBOL(ia64_ivt);
115
116#include <asm/ftrace.h>
117#ifdef CONFIG_FUNCTION_TRACER
118/* mcount is defined in assembly */
119EXPORT_SYMBOL(_mcount);
120#endif
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 10d6cc3fd052..e1983fa025d2 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -174,28 +174,8 @@ config IOMMU_LEAK
174 Add a simple leak tracer to the IOMMU code. This is useful when you 174 Add a simple leak tracer to the IOMMU code. This is useful when you
175 are debugging a buggy device driver that leaks IOMMU mappings. 175 are debugging a buggy device driver that leaks IOMMU mappings.
176 176
177config MMIOTRACE 177config HAVE_MMIOTRACE_SUPPORT
178 bool "Memory mapped IO tracing" 178 def_bool y
179 depends on DEBUG_KERNEL && PCI
180 select TRACING
181 help
182 Mmiotrace traces Memory Mapped I/O access and is meant for
183 debugging and reverse engineering. It is called from the ioremap
184 implementation and works via page faults. Tracing is disabled by
185 default and can be enabled at run-time.
186
187 See Documentation/tracers/mmiotrace.txt.
188 If you are not helping to develop drivers, say N.
189
190config MMIOTRACE_TEST
191 tristate "Test module for mmiotrace"
192 depends on MMIOTRACE && m
193 help
194 This is a dumb module for testing mmiotrace. It is very dangerous
195 as it will write garbage to IO memory starting at a given address.
196 However, it should be safe to use on e.g. unused portion of VRAM.
197
198 Say N, unless you absolutely know what you are doing.
199 179
200# 180#
201# IO delay types: 181# IO delay types:
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index b81125f0bdee..c7da3683f4c5 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -55,7 +55,8 @@ config KVM_AMD
55 55
56config KVM_TRACE 56config KVM_TRACE
57 bool "KVM trace support" 57 bool "KVM trace support"
58 depends on KVM && MARKERS && SYSFS 58 depends on KVM && SYSFS
59 select MARKERS
59 select RELAY 60 select RELAY
60 select DEBUG_FS 61 select DEBUG_FS
61 default n 62 default n
diff --git a/drivers/char/sysrq.c b/drivers/char/sysrq.c
index 33a9351c896d..30659ce9bcf4 100644
--- a/drivers/char/sysrq.c
+++ b/drivers/char/sysrq.c
@@ -283,7 +283,7 @@ static void sysrq_ftrace_dump(int key, struct tty_struct *tty)
283} 283}
284static struct sysrq_key_op sysrq_ftrace_dump_op = { 284static struct sysrq_key_op sysrq_ftrace_dump_op = {
285 .handler = sysrq_ftrace_dump, 285 .handler = sysrq_ftrace_dump,
286 .help_msg = "dumpZ-ftrace-buffer", 286 .help_msg = "dump-ftrace-buffer(Z)",
287 .action_msg = "Dump ftrace buffer", 287 .action_msg = "Dump ftrace buffer",
288 .enable_mask = SYSRQ_ENABLE_DUMP, 288 .enable_mask = SYSRQ_ENABLE_DUMP,
289}; 289};
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 677432b9cb7e..054721487574 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -126,6 +126,10 @@ extern int ftrace_update_ftrace_func(ftrace_func_t func);
126extern void ftrace_caller(void); 126extern void ftrace_caller(void);
127extern void ftrace_call(void); 127extern void ftrace_call(void);
128extern void mcount_call(void); 128extern void mcount_call(void);
129
130#ifndef FTRACE_ADDR
131#define FTRACE_ADDR ((unsigned long)ftrace_caller)
132#endif
129#ifdef CONFIG_FUNCTION_GRAPH_TRACER 133#ifdef CONFIG_FUNCTION_GRAPH_TRACER
130extern void ftrace_graph_caller(void); 134extern void ftrace_graph_caller(void);
131extern int ftrace_enable_ftrace_graph_caller(void); 135extern int ftrace_enable_ftrace_graph_caller(void);
diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h
index 39c3a5eb8ebe..455f9affea9a 100644
--- a/include/linux/slab_def.h
+++ b/include/linux/slab_def.h
@@ -14,6 +14,7 @@
14#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */ 14#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
15#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */ 15#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
16#include <linux/compiler.h> 16#include <linux/compiler.h>
17#include <trace/kmemtrace.h>
17 18
18/* Size description struct for general caches. */ 19/* Size description struct for general caches. */
19struct cache_sizes { 20struct cache_sizes {
@@ -28,8 +29,26 @@ extern struct cache_sizes malloc_sizes[];
28void *kmem_cache_alloc(struct kmem_cache *, gfp_t); 29void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
29void *__kmalloc(size_t size, gfp_t flags); 30void *__kmalloc(size_t size, gfp_t flags);
30 31
31static inline void *kmalloc(size_t size, gfp_t flags) 32#ifdef CONFIG_KMEMTRACE
33extern void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags);
34extern size_t slab_buffer_size(struct kmem_cache *cachep);
35#else
36static __always_inline void *
37kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
32{ 38{
39 return kmem_cache_alloc(cachep, flags);
40}
41static inline size_t slab_buffer_size(struct kmem_cache *cachep)
42{
43 return 0;
44}
45#endif
46
47static __always_inline void *kmalloc(size_t size, gfp_t flags)
48{
49 struct kmem_cache *cachep;
50 void *ret;
51
33 if (__builtin_constant_p(size)) { 52 if (__builtin_constant_p(size)) {
34 int i = 0; 53 int i = 0;
35 54
@@ -50,10 +69,17 @@ static inline void *kmalloc(size_t size, gfp_t flags)
50found: 69found:
51#ifdef CONFIG_ZONE_DMA 70#ifdef CONFIG_ZONE_DMA
52 if (flags & GFP_DMA) 71 if (flags & GFP_DMA)
53 return kmem_cache_alloc(malloc_sizes[i].cs_dmacachep, 72 cachep = malloc_sizes[i].cs_dmacachep;
54 flags); 73 else
55#endif 74#endif
56 return kmem_cache_alloc(malloc_sizes[i].cs_cachep, flags); 75 cachep = malloc_sizes[i].cs_cachep;
76
77 ret = kmem_cache_alloc_notrace(cachep, flags);
78
79 kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_, ret,
80 size, slab_buffer_size(cachep), flags);
81
82 return ret;
57 } 83 }
58 return __kmalloc(size, flags); 84 return __kmalloc(size, flags);
59} 85}
@@ -62,8 +88,25 @@ found:
62extern void *__kmalloc_node(size_t size, gfp_t flags, int node); 88extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
63extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); 89extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
64 90
65static inline void *kmalloc_node(size_t size, gfp_t flags, int node) 91#ifdef CONFIG_KMEMTRACE
92extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
93 gfp_t flags,
94 int nodeid);
95#else
96static __always_inline void *
97kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
98 gfp_t flags,
99 int nodeid)
100{
101 return kmem_cache_alloc_node(cachep, flags, nodeid);
102}
103#endif
104
105static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
66{ 106{
107 struct kmem_cache *cachep;
108 void *ret;
109
67 if (__builtin_constant_p(size)) { 110 if (__builtin_constant_p(size)) {
68 int i = 0; 111 int i = 0;
69 112
@@ -84,11 +127,18 @@ static inline void *kmalloc_node(size_t size, gfp_t flags, int node)
84found: 127found:
85#ifdef CONFIG_ZONE_DMA 128#ifdef CONFIG_ZONE_DMA
86 if (flags & GFP_DMA) 129 if (flags & GFP_DMA)
87 return kmem_cache_alloc_node(malloc_sizes[i].cs_dmacachep, 130 cachep = malloc_sizes[i].cs_dmacachep;
88 flags, node); 131 else
89#endif 132#endif
90 return kmem_cache_alloc_node(malloc_sizes[i].cs_cachep, 133 cachep = malloc_sizes[i].cs_cachep;
91 flags, node); 134
135 ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
136
137 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_,
138 ret, size, slab_buffer_size(cachep),
139 flags, node);
140
141 return ret;
92 } 142 }
93 return __kmalloc_node(size, flags, node); 143 return __kmalloc_node(size, flags, node);
94} 144}
diff --git a/include/linux/slob_def.h b/include/linux/slob_def.h
index 59a3fa476ab9..0ec00b39d006 100644
--- a/include/linux/slob_def.h
+++ b/include/linux/slob_def.h
@@ -3,14 +3,15 @@
3 3
4void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); 4void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
5 5
6static inline void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) 6static __always_inline void *kmem_cache_alloc(struct kmem_cache *cachep,
7 gfp_t flags)
7{ 8{
8 return kmem_cache_alloc_node(cachep, flags, -1); 9 return kmem_cache_alloc_node(cachep, flags, -1);
9} 10}
10 11
11void *__kmalloc_node(size_t size, gfp_t flags, int node); 12void *__kmalloc_node(size_t size, gfp_t flags, int node);
12 13
13static inline void *kmalloc_node(size_t size, gfp_t flags, int node) 14static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
14{ 15{
15 return __kmalloc_node(size, flags, node); 16 return __kmalloc_node(size, flags, node);
16} 17}
@@ -23,12 +24,12 @@ static inline void *kmalloc_node(size_t size, gfp_t flags, int node)
23 * kmalloc is the normal method of allocating memory 24 * kmalloc is the normal method of allocating memory
24 * in the kernel. 25 * in the kernel.
25 */ 26 */
26static inline void *kmalloc(size_t size, gfp_t flags) 27static __always_inline void *kmalloc(size_t size, gfp_t flags)
27{ 28{
28 return __kmalloc_node(size, flags, -1); 29 return __kmalloc_node(size, flags, -1);
29} 30}
30 31
31static inline void *__kmalloc(size_t size, gfp_t flags) 32static __always_inline void *__kmalloc(size_t size, gfp_t flags)
32{ 33{
33 return kmalloc(size, flags); 34 return kmalloc(size, flags);
34} 35}
diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index 2f5c16b1aacd..6b657f7dcb2b 100644
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -10,6 +10,7 @@
10#include <linux/gfp.h> 10#include <linux/gfp.h>
11#include <linux/workqueue.h> 11#include <linux/workqueue.h>
12#include <linux/kobject.h> 12#include <linux/kobject.h>
13#include <trace/kmemtrace.h>
13 14
14enum stat_item { 15enum stat_item {
15 ALLOC_FASTPATH, /* Allocation from cpu slab */ 16 ALLOC_FASTPATH, /* Allocation from cpu slab */
@@ -204,13 +205,31 @@ static __always_inline struct kmem_cache *kmalloc_slab(size_t size)
204void *kmem_cache_alloc(struct kmem_cache *, gfp_t); 205void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
205void *__kmalloc(size_t size, gfp_t flags); 206void *__kmalloc(size_t size, gfp_t flags);
206 207
208#ifdef CONFIG_KMEMTRACE
209extern void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags);
210#else
211static __always_inline void *
212kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
213{
214 return kmem_cache_alloc(s, gfpflags);
215}
216#endif
217
207static __always_inline void *kmalloc_large(size_t size, gfp_t flags) 218static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
208{ 219{
209 return (void *)__get_free_pages(flags | __GFP_COMP, get_order(size)); 220 unsigned int order = get_order(size);
221 void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order);
222
223 kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_, ret,
224 size, PAGE_SIZE << order, flags);
225
226 return ret;
210} 227}
211 228
212static __always_inline void *kmalloc(size_t size, gfp_t flags) 229static __always_inline void *kmalloc(size_t size, gfp_t flags)
213{ 230{
231 void *ret;
232
214 if (__builtin_constant_p(size)) { 233 if (__builtin_constant_p(size)) {
215 if (size > PAGE_SIZE) 234 if (size > PAGE_SIZE)
216 return kmalloc_large(size, flags); 235 return kmalloc_large(size, flags);
@@ -221,7 +240,13 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags)
221 if (!s) 240 if (!s)
222 return ZERO_SIZE_PTR; 241 return ZERO_SIZE_PTR;
223 242
224 return kmem_cache_alloc(s, flags); 243 ret = kmem_cache_alloc_notrace(s, flags);
244
245 kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC,
246 _THIS_IP_, ret,
247 size, s->size, flags);
248
249 return ret;
225 } 250 }
226 } 251 }
227 return __kmalloc(size, flags); 252 return __kmalloc(size, flags);
@@ -231,8 +256,24 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags)
231void *__kmalloc_node(size_t size, gfp_t flags, int node); 256void *__kmalloc_node(size_t size, gfp_t flags, int node);
232void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); 257void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
233 258
259#ifdef CONFIG_KMEMTRACE
260extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
261 gfp_t gfpflags,
262 int node);
263#else
264static __always_inline void *
265kmem_cache_alloc_node_notrace(struct kmem_cache *s,
266 gfp_t gfpflags,
267 int node)
268{
269 return kmem_cache_alloc_node(s, gfpflags, node);
270}
271#endif
272
234static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) 273static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
235{ 274{
275 void *ret;
276
236 if (__builtin_constant_p(size) && 277 if (__builtin_constant_p(size) &&
237 size <= PAGE_SIZE && !(flags & SLUB_DMA)) { 278 size <= PAGE_SIZE && !(flags & SLUB_DMA)) {
238 struct kmem_cache *s = kmalloc_slab(size); 279 struct kmem_cache *s = kmalloc_slab(size);
@@ -240,7 +281,13 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
240 if (!s) 281 if (!s)
241 return ZERO_SIZE_PTR; 282 return ZERO_SIZE_PTR;
242 283
243 return kmem_cache_alloc_node(s, flags, node); 284 ret = kmem_cache_alloc_node_notrace(s, flags, node);
285
286 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
287 _THIS_IP_, ret,
288 size, s->size, flags, node);
289
290 return ret;
244 } 291 }
245 return __kmalloc_node(size, flags, node); 292 return __kmalloc_node(size, flags, node);
246} 293}
diff --git a/include/trace/kmemtrace.h b/include/trace/kmemtrace.h
new file mode 100644
index 000000000000..ad8b7857855a
--- /dev/null
+++ b/include/trace/kmemtrace.h
@@ -0,0 +1,75 @@
1/*
2 * Copyright (C) 2008 Eduard - Gabriel Munteanu
3 *
4 * This file is released under GPL version 2.
5 */
6
7#ifndef _LINUX_KMEMTRACE_H
8#define _LINUX_KMEMTRACE_H
9
10#ifdef __KERNEL__
11
12#include <linux/types.h>
13#include <linux/marker.h>
14
15enum kmemtrace_type_id {
16 KMEMTRACE_TYPE_KMALLOC = 0, /* kmalloc() or kfree(). */
17 KMEMTRACE_TYPE_CACHE, /* kmem_cache_*(). */
18 KMEMTRACE_TYPE_PAGES, /* __get_free_pages() and friends. */
19};
20
21#ifdef CONFIG_KMEMTRACE
22
23extern void kmemtrace_init(void);
24
25extern void kmemtrace_mark_alloc_node(enum kmemtrace_type_id type_id,
26 unsigned long call_site,
27 const void *ptr,
28 size_t bytes_req,
29 size_t bytes_alloc,
30 gfp_t gfp_flags,
31 int node);
32
33extern void kmemtrace_mark_free(enum kmemtrace_type_id type_id,
34 unsigned long call_site,
35 const void *ptr);
36
37#else /* CONFIG_KMEMTRACE */
38
39static inline void kmemtrace_init(void)
40{
41}
42
43static inline void kmemtrace_mark_alloc_node(enum kmemtrace_type_id type_id,
44 unsigned long call_site,
45 const void *ptr,
46 size_t bytes_req,
47 size_t bytes_alloc,
48 gfp_t gfp_flags,
49 int node)
50{
51}
52
53static inline void kmemtrace_mark_free(enum kmemtrace_type_id type_id,
54 unsigned long call_site,
55 const void *ptr)
56{
57}
58
59#endif /* CONFIG_KMEMTRACE */
60
61static inline void kmemtrace_mark_alloc(enum kmemtrace_type_id type_id,
62 unsigned long call_site,
63 const void *ptr,
64 size_t bytes_req,
65 size_t bytes_alloc,
66 gfp_t gfp_flags)
67{
68 kmemtrace_mark_alloc_node(type_id, call_site, ptr,
69 bytes_req, bytes_alloc, gfp_flags, -1);
70}
71
72#endif /* __KERNEL__ */
73
74#endif /* _LINUX_KMEMTRACE_H */
75
diff --git a/include/trace/workqueue.h b/include/trace/workqueue.h
new file mode 100644
index 000000000000..867829df4571
--- /dev/null
+++ b/include/trace/workqueue.h
@@ -0,0 +1,25 @@
1#ifndef __TRACE_WORKQUEUE_H
2#define __TRACE_WORKQUEUE_H
3
4#include <linux/tracepoint.h>
5#include <linux/workqueue.h>
6#include <linux/sched.h>
7
8DECLARE_TRACE(workqueue_insertion,
9 TPPROTO(struct task_struct *wq_thread, struct work_struct *work),
10 TPARGS(wq_thread, work));
11
12DECLARE_TRACE(workqueue_execution,
13 TPPROTO(struct task_struct *wq_thread, struct work_struct *work),
14 TPARGS(wq_thread, work));
15
16/* Trace the creation of one workqueue thread on a cpu */
17DECLARE_TRACE(workqueue_creation,
18 TPPROTO(struct task_struct *wq_thread, int cpu),
19 TPARGS(wq_thread, cpu));
20
21DECLARE_TRACE(workqueue_destruction,
22 TPPROTO(struct task_struct *wq_thread),
23 TPARGS(wq_thread));
24
25#endif /* __TRACE_WORKQUEUE_H */
diff --git a/init/main.c b/init/main.c
index 844209453c02..db7974ff7a0a 100644
--- a/init/main.c
+++ b/init/main.c
@@ -70,6 +70,7 @@
70#include <asm/setup.h> 70#include <asm/setup.h>
71#include <asm/sections.h> 71#include <asm/sections.h>
72#include <asm/cacheflush.h> 72#include <asm/cacheflush.h>
73#include <trace/kmemtrace.h>
73 74
74#ifdef CONFIG_X86_LOCAL_APIC 75#ifdef CONFIG_X86_LOCAL_APIC
75#include <asm/smp.h> 76#include <asm/smp.h>
@@ -641,6 +642,7 @@ asmlinkage void __init start_kernel(void)
641 enable_debug_pagealloc(); 642 enable_debug_pagealloc();
642 cpu_hotplug_init(); 643 cpu_hotplug_init();
643 kmem_cache_init(); 644 kmem_cache_init();
645 kmemtrace_init();
644 debug_objects_mem_init(); 646 debug_objects_mem_init();
645 idr_init_cache(); 647 idr_init_cache();
646 setup_per_cpu_pageset(); 648 setup_per_cpu_pageset();
diff --git a/kernel/relay.c b/kernel/relay.c
index 09ac2008f77b..d06450670c86 100644
--- a/kernel/relay.c
+++ b/kernel/relay.c
@@ -675,9 +675,7 @@ int relay_late_setup_files(struct rchan *chan,
675 */ 675 */
676 for_each_online_cpu(i) { 676 for_each_online_cpu(i) {
677 if (unlikely(!chan->buf[i])) { 677 if (unlikely(!chan->buf[i])) {
678 printk(KERN_ERR "relay_late_setup_files: CPU %u " 678 WARN_ONCE(1, KERN_ERR "CPU has no buffer!\n");
679 "has no buffer, it must have!\n", i);
680 BUG();
681 err = -EINVAL; 679 err = -EINVAL;
682 break; 680 break;
683 } 681 }
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index e2a4ff6fc3a6..dde1d46f77e5 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -264,6 +264,38 @@ config HW_BRANCH_TRACER
264 This tracer records all branches on the system in a circular 264 This tracer records all branches on the system in a circular
265 buffer giving access to the last N branches for each cpu. 265 buffer giving access to the last N branches for each cpu.
266 266
267config KMEMTRACE
268 bool "Trace SLAB allocations"
269 select TRACING
270 help
271 kmemtrace provides tracing for slab allocator functions, such as
272 kmalloc, kfree, kmem_cache_alloc, kmem_cache_free etc.. Collected
273 data is then fed to the userspace application in order to analyse
274 allocation hotspots, internal fragmentation and so on, making it
275 possible to see how well an allocator performs, as well as debug
276 and profile kernel code.
277
278 This requires an userspace application to use. See
279 Documentation/vm/kmemtrace.txt for more information.
280
281 Saying Y will make the kernel somewhat larger and slower. However,
282 if you disable kmemtrace at run-time or boot-time, the performance
283 impact is minimal (depending on the arch the kernel is built for).
284
285 If unsure, say N.
286
287config WORKQUEUE_TRACER
288 bool "Trace workqueues"
289 select TRACING
290 help
291 The workqueue tracer provides some statistical informations
292 about each cpu workqueue thread such as the number of the
293 works inserted and executed since their creation. It can help
294 to evaluate the amount of work each of them have to perform.
295 For example it can help a developer to decide whether he should
296 choose a per cpu workqueue instead of a singlethreaded one.
297
298
267config DYNAMIC_FTRACE 299config DYNAMIC_FTRACE
268 bool "enable/disable ftrace tracepoints dynamically" 300 bool "enable/disable ftrace tracepoints dynamically"
269 depends on FUNCTION_TRACER 301 depends on FUNCTION_TRACER
@@ -302,4 +334,27 @@ config FTRACE_STARTUP_TEST
302 functioning properly. It will do tests on all the configured 334 functioning properly. It will do tests on all the configured
303 tracers of ftrace. 335 tracers of ftrace.
304 336
337config MMIOTRACE
338 bool "Memory mapped IO tracing"
339 depends on HAVE_MMIOTRACE_SUPPORT && DEBUG_KERNEL && PCI
340 select TRACING
341 help
342 Mmiotrace traces Memory Mapped I/O access and is meant for
343 debugging and reverse engineering. It is called from the ioremap
344 implementation and works via page faults. Tracing is disabled by
345 default and can be enabled at run-time.
346
347 See Documentation/tracers/mmiotrace.txt.
348 If you are not helping to develop drivers, say N.
349
350config MMIOTRACE_TEST
351 tristate "Test module for mmiotrace"
352 depends on MMIOTRACE && m
353 help
354 This is a dumb module for testing mmiotrace. It is very dangerous
355 as it will write garbage to IO memory starting at a given address.
356 However, it should be safe to use on e.g. unused portion of VRAM.
357
358 Say N, unless you absolutely know what you are doing.
359
305endmenu 360endmenu
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index 349d5a93653f..f76d48f3527d 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -19,6 +19,8 @@ obj-$(CONFIG_FUNCTION_TRACER) += libftrace.o
19obj-$(CONFIG_RING_BUFFER) += ring_buffer.o 19obj-$(CONFIG_RING_BUFFER) += ring_buffer.o
20 20
21obj-$(CONFIG_TRACING) += trace.o 21obj-$(CONFIG_TRACING) += trace.o
22obj-$(CONFIG_TRACING) += trace_output.o
23obj-$(CONFIG_TRACING) += trace_stat.o
22obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o 24obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o
23obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o 25obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o
24obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o 26obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o
@@ -33,5 +35,7 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o
33obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o 35obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o
34obj-$(CONFIG_HW_BRANCH_TRACER) += trace_hw_branches.o 36obj-$(CONFIG_HW_BRANCH_TRACER) += trace_hw_branches.o
35obj-$(CONFIG_POWER_TRACER) += trace_power.o 37obj-$(CONFIG_POWER_TRACER) += trace_power.o
38obj-$(CONFIG_KMEMTRACE) += kmemtrace.o
39obj-$(CONFIG_WORKQUEUE_TRACER) += trace_workqueue.o
36 40
37libftrace-y := ftrace.o 41libftrace-y := ftrace.o
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 2f32969c09df..7e9a20b69939 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -17,6 +17,7 @@
17#include <linux/clocksource.h> 17#include <linux/clocksource.h>
18#include <linux/kallsyms.h> 18#include <linux/kallsyms.h>
19#include <linux/seq_file.h> 19#include <linux/seq_file.h>
20#include <linux/suspend.h>
20#include <linux/debugfs.h> 21#include <linux/debugfs.h>
21#include <linux/hardirq.h> 22#include <linux/hardirq.h>
22#include <linux/kthread.h> 23#include <linux/kthread.h>
@@ -263,14 +264,6 @@ static void ftrace_update_pid_func(void)
263# error Dynamic ftrace depends on MCOUNT_RECORD 264# error Dynamic ftrace depends on MCOUNT_RECORD
264#endif 265#endif
265 266
266/*
267 * Since MCOUNT_ADDR may point to mcount itself, we do not want
268 * to get it confused by reading a reference in the code as we
269 * are parsing on objcopy output of text. Use a variable for
270 * it instead.
271 */
272static unsigned long mcount_addr = MCOUNT_ADDR;
273
274enum { 267enum {
275 FTRACE_ENABLE_CALLS = (1 << 0), 268 FTRACE_ENABLE_CALLS = (1 << 0),
276 FTRACE_DISABLE_CALLS = (1 << 1), 269 FTRACE_DISABLE_CALLS = (1 << 1),
@@ -289,7 +282,7 @@ static DEFINE_MUTEX(ftrace_regex_lock);
289 282
290struct ftrace_page { 283struct ftrace_page {
291 struct ftrace_page *next; 284 struct ftrace_page *next;
292 unsigned long index; 285 int index;
293 struct dyn_ftrace records[]; 286 struct dyn_ftrace records[];
294}; 287};
295 288
@@ -463,7 +456,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
463 unsigned long ip, fl; 456 unsigned long ip, fl;
464 unsigned long ftrace_addr; 457 unsigned long ftrace_addr;
465 458
466 ftrace_addr = (unsigned long)ftrace_caller; 459 ftrace_addr = (unsigned long)FTRACE_ADDR;
467 460
468 ip = rec->ip; 461 ip = rec->ip;
469 462
@@ -575,7 +568,7 @@ ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
575 568
576 ip = rec->ip; 569 ip = rec->ip;
577 570
578 ret = ftrace_make_nop(mod, rec, mcount_addr); 571 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
579 if (ret) { 572 if (ret) {
580 ftrace_bug(ret, ip); 573 ftrace_bug(ret, ip);
581 rec->flags |= FTRACE_FL_FAILED; 574 rec->flags |= FTRACE_FL_FAILED;
@@ -786,7 +779,7 @@ enum {
786 779
787struct ftrace_iterator { 780struct ftrace_iterator {
788 struct ftrace_page *pg; 781 struct ftrace_page *pg;
789 unsigned idx; 782 int idx;
790 unsigned flags; 783 unsigned flags;
791 unsigned char buffer[FTRACE_BUFF_MAX+1]; 784 unsigned char buffer[FTRACE_BUFF_MAX+1];
792 unsigned buffer_idx; 785 unsigned buffer_idx;
@@ -1902,7 +1895,7 @@ int register_ftrace_function(struct ftrace_ops *ops)
1902} 1895}
1903 1896
1904/** 1897/**
1905 * unregister_ftrace_function - unresgister a function for profiling. 1898 * unregister_ftrace_function - unregister a function for profiling.
1906 * @ops - ops structure that holds the function to unregister 1899 * @ops - ops structure that holds the function to unregister
1907 * 1900 *
1908 * Unregister a function that was added to be called by ftrace profiling. 1901 * Unregister a function that was added to be called by ftrace profiling.
@@ -1965,6 +1958,7 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
1965#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1958#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1966 1959
1967static atomic_t ftrace_graph_active; 1960static atomic_t ftrace_graph_active;
1961static struct notifier_block ftrace_suspend_notifier;
1968 1962
1969int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) 1963int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
1970{ 1964{
@@ -2043,6 +2037,27 @@ static int start_graph_tracing(void)
2043 return ret; 2037 return ret;
2044} 2038}
2045 2039
2040/*
2041 * Hibernation protection.
2042 * The state of the current task is too much unstable during
2043 * suspend/restore to disk. We want to protect against that.
2044 */
2045static int
2046ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
2047 void *unused)
2048{
2049 switch (state) {
2050 case PM_HIBERNATION_PREPARE:
2051 pause_graph_tracing();
2052 break;
2053
2054 case PM_POST_HIBERNATION:
2055 unpause_graph_tracing();
2056 break;
2057 }
2058 return NOTIFY_DONE;
2059}
2060
2046int register_ftrace_graph(trace_func_graph_ret_t retfunc, 2061int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2047 trace_func_graph_ent_t entryfunc) 2062 trace_func_graph_ent_t entryfunc)
2048{ 2063{
@@ -2050,6 +2065,9 @@ int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2050 2065
2051 mutex_lock(&ftrace_sysctl_lock); 2066 mutex_lock(&ftrace_sysctl_lock);
2052 2067
2068 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2069 register_pm_notifier(&ftrace_suspend_notifier);
2070
2053 atomic_inc(&ftrace_graph_active); 2071 atomic_inc(&ftrace_graph_active);
2054 ret = start_graph_tracing(); 2072 ret = start_graph_tracing();
2055 if (ret) { 2073 if (ret) {
@@ -2075,6 +2093,7 @@ void unregister_ftrace_graph(void)
2075 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 2093 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2076 ftrace_graph_entry = ftrace_graph_entry_stub; 2094 ftrace_graph_entry = ftrace_graph_entry_stub;
2077 ftrace_shutdown(FTRACE_STOP_FUNC_RET); 2095 ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2096 unregister_pm_notifier(&ftrace_suspend_notifier);
2078 2097
2079 mutex_unlock(&ftrace_sysctl_lock); 2098 mutex_unlock(&ftrace_sysctl_lock);
2080} 2099}
diff --git a/kernel/trace/kmemtrace.c b/kernel/trace/kmemtrace.c
new file mode 100644
index 000000000000..7ebc58cee3bd
--- /dev/null
+++ b/kernel/trace/kmemtrace.c
@@ -0,0 +1,350 @@
1/*
2 * Memory allocator tracing
3 *
4 * Copyright (C) 2008 Eduard - Gabriel Munteanu
5 * Copyright (C) 2008 Pekka Enberg <penberg@cs.helsinki.fi>
6 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
7 */
8
9#include <linux/dcache.h>
10#include <linux/debugfs.h>
11#include <linux/fs.h>
12#include <linux/seq_file.h>
13#include <trace/kmemtrace.h>
14
15#include "trace.h"
16#include "trace_output.h"
17
18/* Select an alternative, minimalistic output than the original one */
19#define TRACE_KMEM_OPT_MINIMAL 0x1
20
21static struct tracer_opt kmem_opts[] = {
22 /* Default disable the minimalistic output */
23 { TRACER_OPT(kmem_minimalistic, TRACE_KMEM_OPT_MINIMAL) },
24 { }
25};
26
27static struct tracer_flags kmem_tracer_flags = {
28 .val = 0,
29 .opts = kmem_opts
30};
31
32
33static bool kmem_tracing_enabled __read_mostly;
34static struct trace_array *kmemtrace_array;
35
36static int kmem_trace_init(struct trace_array *tr)
37{
38 int cpu;
39 kmemtrace_array = tr;
40
41 for_each_cpu_mask(cpu, cpu_possible_map)
42 tracing_reset(tr, cpu);
43
44 kmem_tracing_enabled = true;
45
46 return 0;
47}
48
49static void kmem_trace_reset(struct trace_array *tr)
50{
51 kmem_tracing_enabled = false;
52}
53
54static void kmemtrace_headers(struct seq_file *s)
55{
56 /* Don't need headers for the original kmemtrace output */
57 if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL))
58 return;
59
60 seq_printf(s, "#\n");
61 seq_printf(s, "# ALLOC TYPE REQ GIVEN FLAGS "
62 " POINTER NODE CALLER\n");
63 seq_printf(s, "# FREE | | | | "
64 " | | | |\n");
65 seq_printf(s, "# |\n\n");
66}
67
68/*
69 * The two following functions give the original output from kmemtrace,
70 * or something close to....perhaps they need some missing things
71 */
72static enum print_line_t
73kmemtrace_print_alloc_original(struct trace_iterator *iter,
74 struct kmemtrace_alloc_entry *entry)
75{
76 struct trace_seq *s = &iter->seq;
77 int ret;
78
79 /* Taken from the old linux/kmemtrace.h */
80 ret = trace_seq_printf(s, "type_id %d call_site %lu ptr %lu "
81 "bytes_req %lu bytes_alloc %lu gfp_flags %lu node %d\n",
82 entry->type_id, entry->call_site, (unsigned long) entry->ptr,
83 (unsigned long) entry->bytes_req, (unsigned long) entry->bytes_alloc,
84 (unsigned long) entry->gfp_flags, entry->node);
85
86 if (!ret)
87 return TRACE_TYPE_PARTIAL_LINE;
88
89 return TRACE_TYPE_HANDLED;
90}
91
92static enum print_line_t
93kmemtrace_print_free_original(struct trace_iterator *iter,
94 struct kmemtrace_free_entry *entry)
95{
96 struct trace_seq *s = &iter->seq;
97 int ret;
98
99 /* Taken from the old linux/kmemtrace.h */
100 ret = trace_seq_printf(s, "type_id %d call_site %lu ptr %lu\n",
101 entry->type_id, entry->call_site, (unsigned long) entry->ptr);
102
103 if (!ret)
104 return TRACE_TYPE_PARTIAL_LINE;
105
106 return TRACE_TYPE_HANDLED;
107}
108
109
110/* The two other following provide a more minimalistic output */
111static enum print_line_t
112kmemtrace_print_alloc_compress(struct trace_iterator *iter,
113 struct kmemtrace_alloc_entry *entry)
114{
115 struct trace_seq *s = &iter->seq;
116 int ret;
117
118 /* Alloc entry */
119 ret = trace_seq_printf(s, " + ");
120 if (!ret)
121 return TRACE_TYPE_PARTIAL_LINE;
122
123 /* Type */
124 switch (entry->type_id) {
125 case KMEMTRACE_TYPE_KMALLOC:
126 ret = trace_seq_printf(s, "K ");
127 break;
128 case KMEMTRACE_TYPE_CACHE:
129 ret = trace_seq_printf(s, "C ");
130 break;
131 case KMEMTRACE_TYPE_PAGES:
132 ret = trace_seq_printf(s, "P ");
133 break;
134 default:
135 ret = trace_seq_printf(s, "? ");
136 }
137
138 if (!ret)
139 return TRACE_TYPE_PARTIAL_LINE;
140
141 /* Requested */
142 ret = trace_seq_printf(s, "%4ld ", entry->bytes_req);
143 if (!ret)
144 return TRACE_TYPE_PARTIAL_LINE;
145
146 /* Allocated */
147 ret = trace_seq_printf(s, "%4ld ", entry->bytes_alloc);
148 if (!ret)
149 return TRACE_TYPE_PARTIAL_LINE;
150
151 /* Flags
152 * TODO: would be better to see the name of the GFP flag names
153 */
154 ret = trace_seq_printf(s, "%08x ", entry->gfp_flags);
155 if (!ret)
156 return TRACE_TYPE_PARTIAL_LINE;
157
158 /* Pointer to allocated */
159 ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
160 if (!ret)
161 return TRACE_TYPE_PARTIAL_LINE;
162
163 /* Node */
164 ret = trace_seq_printf(s, "%4d ", entry->node);
165 if (!ret)
166 return TRACE_TYPE_PARTIAL_LINE;
167
168 /* Call site */
169 ret = seq_print_ip_sym(s, entry->call_site, 0);
170 if (!ret)
171 return TRACE_TYPE_PARTIAL_LINE;
172
173 if (!trace_seq_printf(s, "\n"))
174 return TRACE_TYPE_PARTIAL_LINE;
175
176 return TRACE_TYPE_HANDLED;
177}
178
179static enum print_line_t
180kmemtrace_print_free_compress(struct trace_iterator *iter,
181 struct kmemtrace_free_entry *entry)
182{
183 struct trace_seq *s = &iter->seq;
184 int ret;
185
186 /* Free entry */
187 ret = trace_seq_printf(s, " - ");
188 if (!ret)
189 return TRACE_TYPE_PARTIAL_LINE;
190
191 /* Type */
192 switch (entry->type_id) {
193 case KMEMTRACE_TYPE_KMALLOC:
194 ret = trace_seq_printf(s, "K ");
195 break;
196 case KMEMTRACE_TYPE_CACHE:
197 ret = trace_seq_printf(s, "C ");
198 break;
199 case KMEMTRACE_TYPE_PAGES:
200 ret = trace_seq_printf(s, "P ");
201 break;
202 default:
203 ret = trace_seq_printf(s, "? ");
204 }
205
206 if (!ret)
207 return TRACE_TYPE_PARTIAL_LINE;
208
209 /* Skip requested/allocated/flags */
210 ret = trace_seq_printf(s, " ");
211 if (!ret)
212 return TRACE_TYPE_PARTIAL_LINE;
213
214 /* Pointer to allocated */
215 ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
216 if (!ret)
217 return TRACE_TYPE_PARTIAL_LINE;
218
219 /* Skip node */
220 ret = trace_seq_printf(s, " ");
221 if (!ret)
222 return TRACE_TYPE_PARTIAL_LINE;
223
224 /* Call site */
225 ret = seq_print_ip_sym(s, entry->call_site, 0);
226 if (!ret)
227 return TRACE_TYPE_PARTIAL_LINE;
228
229 if (!trace_seq_printf(s, "\n"))
230 return TRACE_TYPE_PARTIAL_LINE;
231
232 return TRACE_TYPE_HANDLED;
233}
234
235static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter)
236{
237 struct trace_entry *entry = iter->ent;
238
239 switch (entry->type) {
240 case TRACE_KMEM_ALLOC: {
241 struct kmemtrace_alloc_entry *field;
242 trace_assign_type(field, entry);
243 if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
244 return kmemtrace_print_alloc_compress(iter, field);
245 else
246 return kmemtrace_print_alloc_original(iter, field);
247 }
248
249 case TRACE_KMEM_FREE: {
250 struct kmemtrace_free_entry *field;
251 trace_assign_type(field, entry);
252 if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
253 return kmemtrace_print_free_compress(iter, field);
254 else
255 return kmemtrace_print_free_original(iter, field);
256 }
257
258 default:
259 return TRACE_TYPE_UNHANDLED;
260 }
261}
262
263/* Trace allocations */
264void kmemtrace_mark_alloc_node(enum kmemtrace_type_id type_id,
265 unsigned long call_site,
266 const void *ptr,
267 size_t bytes_req,
268 size_t bytes_alloc,
269 gfp_t gfp_flags,
270 int node)
271{
272 struct ring_buffer_event *event;
273 struct kmemtrace_alloc_entry *entry;
274 struct trace_array *tr = kmemtrace_array;
275 unsigned long irq_flags;
276
277 if (!kmem_tracing_enabled)
278 return;
279
280 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
281 &irq_flags);
282 if (!event)
283 return;
284 entry = ring_buffer_event_data(event);
285 tracing_generic_entry_update(&entry->ent, 0, 0);
286
287 entry->ent.type = TRACE_KMEM_ALLOC;
288 entry->call_site = call_site;
289 entry->ptr = ptr;
290 entry->bytes_req = bytes_req;
291 entry->bytes_alloc = bytes_alloc;
292 entry->gfp_flags = gfp_flags;
293 entry->node = node;
294
295 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
296
297 trace_wake_up();
298}
299EXPORT_SYMBOL(kmemtrace_mark_alloc_node);
300
301void kmemtrace_mark_free(enum kmemtrace_type_id type_id,
302 unsigned long call_site,
303 const void *ptr)
304{
305 struct ring_buffer_event *event;
306 struct kmemtrace_free_entry *entry;
307 struct trace_array *tr = kmemtrace_array;
308 unsigned long irq_flags;
309
310 if (!kmem_tracing_enabled)
311 return;
312
313 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
314 &irq_flags);
315 if (!event)
316 return;
317 entry = ring_buffer_event_data(event);
318 tracing_generic_entry_update(&entry->ent, 0, 0);
319
320 entry->ent.type = TRACE_KMEM_FREE;
321 entry->type_id = type_id;
322 entry->call_site = call_site;
323 entry->ptr = ptr;
324
325 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
326
327 trace_wake_up();
328}
329EXPORT_SYMBOL(kmemtrace_mark_free);
330
331static struct tracer kmem_tracer __read_mostly = {
332 .name = "kmemtrace",
333 .init = kmem_trace_init,
334 .reset = kmem_trace_reset,
335 .print_line = kmemtrace_print_line,
336 .print_header = kmemtrace_headers,
337 .flags = &kmem_tracer_flags
338};
339
340void kmemtrace_init(void)
341{
342 /* earliest opportunity to start kmem tracing */
343}
344
345static int __init init_kmem_tracer(void)
346{
347 return register_tracer(&kmem_tracer);
348}
349
350device_initcall(init_kmem_tracer);
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 8b0daf0662ef..0b9de5a3d699 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -123,8 +123,7 @@ void ring_buffer_normalize_time_stamp(int cpu, u64 *ts)
123EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp); 123EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
124 124
125#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event)) 125#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event))
126#define RB_ALIGNMENT_SHIFT 2 126#define RB_ALIGNMENT 4U
127#define RB_ALIGNMENT (1 << RB_ALIGNMENT_SHIFT)
128#define RB_MAX_SMALL_DATA 28 127#define RB_MAX_SMALL_DATA 28
129 128
130enum { 129enum {
@@ -133,7 +132,7 @@ enum {
133}; 132};
134 133
135/* inline for ring buffer fast paths */ 134/* inline for ring buffer fast paths */
136static inline unsigned 135static unsigned
137rb_event_length(struct ring_buffer_event *event) 136rb_event_length(struct ring_buffer_event *event)
138{ 137{
139 unsigned length; 138 unsigned length;
@@ -151,7 +150,7 @@ rb_event_length(struct ring_buffer_event *event)
151 150
152 case RINGBUF_TYPE_DATA: 151 case RINGBUF_TYPE_DATA:
153 if (event->len) 152 if (event->len)
154 length = event->len << RB_ALIGNMENT_SHIFT; 153 length = event->len * RB_ALIGNMENT;
155 else 154 else
156 length = event->array[0]; 155 length = event->array[0];
157 return length + RB_EVNT_HDR_SIZE; 156 return length + RB_EVNT_HDR_SIZE;
@@ -179,7 +178,7 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event)
179EXPORT_SYMBOL_GPL(ring_buffer_event_length); 178EXPORT_SYMBOL_GPL(ring_buffer_event_length);
180 179
181/* inline for ring buffer fast paths */ 180/* inline for ring buffer fast paths */
182static inline void * 181static void *
183rb_event_data(struct ring_buffer_event *event) 182rb_event_data(struct ring_buffer_event *event)
184{ 183{
185 BUG_ON(event->type != RINGBUF_TYPE_DATA); 184 BUG_ON(event->type != RINGBUF_TYPE_DATA);
@@ -229,10 +228,9 @@ static void rb_init_page(struct buffer_data_page *bpage)
229 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing 228 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
230 * this issue out. 229 * this issue out.
231 */ 230 */
232static inline void free_buffer_page(struct buffer_page *bpage) 231static void free_buffer_page(struct buffer_page *bpage)
233{ 232{
234 if (bpage->page) 233 free_page((unsigned long)bpage->page);
235 free_page((unsigned long)bpage->page);
236 kfree(bpage); 234 kfree(bpage);
237} 235}
238 236
@@ -811,7 +809,7 @@ rb_event_index(struct ring_buffer_event *event)
811 return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE); 809 return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE);
812} 810}
813 811
814static inline int 812static int
815rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer, 813rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
816 struct ring_buffer_event *event) 814 struct ring_buffer_event *event)
817{ 815{
@@ -825,7 +823,7 @@ rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
825 rb_commit_index(cpu_buffer) == index; 823 rb_commit_index(cpu_buffer) == index;
826} 824}
827 825
828static inline void 826static void
829rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer, 827rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
830 struct ring_buffer_event *event) 828 struct ring_buffer_event *event)
831{ 829{
@@ -850,7 +848,7 @@ rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
850 local_set(&cpu_buffer->commit_page->page->commit, index); 848 local_set(&cpu_buffer->commit_page->page->commit, index);
851} 849}
852 850
853static inline void 851static void
854rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) 852rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
855{ 853{
856 /* 854 /*
@@ -896,7 +894,7 @@ static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
896 cpu_buffer->reader_page->read = 0; 894 cpu_buffer->reader_page->read = 0;
897} 895}
898 896
899static inline void rb_inc_iter(struct ring_buffer_iter *iter) 897static void rb_inc_iter(struct ring_buffer_iter *iter)
900{ 898{
901 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 899 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
902 900
@@ -926,7 +924,7 @@ static inline void rb_inc_iter(struct ring_buffer_iter *iter)
926 * and with this, we can determine what to place into the 924 * and with this, we can determine what to place into the
927 * data field. 925 * data field.
928 */ 926 */
929static inline void 927static void
930rb_update_event(struct ring_buffer_event *event, 928rb_update_event(struct ring_buffer_event *event,
931 unsigned type, unsigned length) 929 unsigned type, unsigned length)
932{ 930{
@@ -938,15 +936,11 @@ rb_update_event(struct ring_buffer_event *event,
938 break; 936 break;
939 937
940 case RINGBUF_TYPE_TIME_EXTEND: 938 case RINGBUF_TYPE_TIME_EXTEND:
941 event->len = 939 event->len = DIV_ROUND_UP(RB_LEN_TIME_EXTEND, RB_ALIGNMENT);
942 (RB_LEN_TIME_EXTEND + (RB_ALIGNMENT-1))
943 >> RB_ALIGNMENT_SHIFT;
944 break; 940 break;
945 941
946 case RINGBUF_TYPE_TIME_STAMP: 942 case RINGBUF_TYPE_TIME_STAMP:
947 event->len = 943 event->len = DIV_ROUND_UP(RB_LEN_TIME_STAMP, RB_ALIGNMENT);
948 (RB_LEN_TIME_STAMP + (RB_ALIGNMENT-1))
949 >> RB_ALIGNMENT_SHIFT;
950 break; 944 break;
951 945
952 case RINGBUF_TYPE_DATA: 946 case RINGBUF_TYPE_DATA:
@@ -955,16 +949,14 @@ rb_update_event(struct ring_buffer_event *event,
955 event->len = 0; 949 event->len = 0;
956 event->array[0] = length; 950 event->array[0] = length;
957 } else 951 } else
958 event->len = 952 event->len = DIV_ROUND_UP(length, RB_ALIGNMENT);
959 (length + (RB_ALIGNMENT-1))
960 >> RB_ALIGNMENT_SHIFT;
961 break; 953 break;
962 default: 954 default:
963 BUG(); 955 BUG();
964 } 956 }
965} 957}
966 958
967static inline unsigned rb_calculate_event_length(unsigned length) 959static unsigned rb_calculate_event_length(unsigned length)
968{ 960{
969 struct ring_buffer_event event; /* Used only for sizeof array */ 961 struct ring_buffer_event event; /* Used only for sizeof array */
970 962
@@ -1025,12 +1017,8 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1025 } 1017 }
1026 1018
1027 if (next_page == head_page) { 1019 if (next_page == head_page) {
1028 if (!(buffer->flags & RB_FL_OVERWRITE)) { 1020 if (!(buffer->flags & RB_FL_OVERWRITE))
1029 /* reset write */
1030 if (tail <= BUF_PAGE_SIZE)
1031 local_set(&tail_page->write, tail);
1032 goto out_unlock; 1021 goto out_unlock;
1033 }
1034 1022
1035 /* tail_page has not moved yet? */ 1023 /* tail_page has not moved yet? */
1036 if (tail_page == cpu_buffer->tail_page) { 1024 if (tail_page == cpu_buffer->tail_page) {
@@ -1105,6 +1093,10 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1105 return event; 1093 return event;
1106 1094
1107 out_unlock: 1095 out_unlock:
1096 /* reset write */
1097 if (tail <= BUF_PAGE_SIZE)
1098 local_set(&tail_page->write, tail);
1099
1108 __raw_spin_unlock(&cpu_buffer->lock); 1100 __raw_spin_unlock(&cpu_buffer->lock);
1109 local_irq_restore(flags); 1101 local_irq_restore(flags);
1110 return NULL; 1102 return NULL;
@@ -1438,7 +1430,7 @@ int ring_buffer_write(struct ring_buffer *buffer,
1438} 1430}
1439EXPORT_SYMBOL_GPL(ring_buffer_write); 1431EXPORT_SYMBOL_GPL(ring_buffer_write);
1440 1432
1441static inline int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer) 1433static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
1442{ 1434{
1443 struct buffer_page *reader = cpu_buffer->reader_page; 1435 struct buffer_page *reader = cpu_buffer->reader_page;
1444 struct buffer_page *head = cpu_buffer->head_page; 1436 struct buffer_page *head = cpu_buffer->head_page;
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index c580233add95..220c264e3111 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -37,10 +37,11 @@
37#include <linux/irqflags.h> 37#include <linux/irqflags.h>
38 38
39#include "trace.h" 39#include "trace.h"
40#include "trace_output.h"
40 41
41#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE) 42#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
42 43
43unsigned long __read_mostly tracing_max_latency = (cycle_t)ULONG_MAX; 44unsigned long __read_mostly tracing_max_latency;
44unsigned long __read_mostly tracing_thresh; 45unsigned long __read_mostly tracing_thresh;
45 46
46/* 47/*
@@ -186,9 +187,6 @@ int tracing_is_enabled(void)
186 return tracer_enabled; 187 return tracer_enabled;
187} 188}
188 189
189/* function tracing enabled */
190int ftrace_function_enabled;
191
192/* 190/*
193 * trace_buf_size is the size in bytes that is allocated 191 * trace_buf_size is the size in bytes that is allocated
194 * for a buffer. Note, the number of bytes is always rounded 192 * for a buffer. Note, the number of bytes is always rounded
@@ -329,132 +327,6 @@ __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
329 tracing_record_cmdline(current); 327 tracing_record_cmdline(current);
330} 328}
331 329
332/**
333 * trace_seq_printf - sequence printing of trace information
334 * @s: trace sequence descriptor
335 * @fmt: printf format string
336 *
337 * The tracer may use either sequence operations or its own
338 * copy to user routines. To simplify formating of a trace
339 * trace_seq_printf is used to store strings into a special
340 * buffer (@s). Then the output may be either used by
341 * the sequencer or pulled into another buffer.
342 */
343int
344trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
345{
346 int len = (PAGE_SIZE - 1) - s->len;
347 va_list ap;
348 int ret;
349
350 if (!len)
351 return 0;
352
353 va_start(ap, fmt);
354 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
355 va_end(ap);
356
357 /* If we can't write it all, don't bother writing anything */
358 if (ret >= len)
359 return 0;
360
361 s->len += ret;
362
363 return len;
364}
365
366/**
367 * trace_seq_puts - trace sequence printing of simple string
368 * @s: trace sequence descriptor
369 * @str: simple string to record
370 *
371 * The tracer may use either the sequence operations or its own
372 * copy to user routines. This function records a simple string
373 * into a special buffer (@s) for later retrieval by a sequencer
374 * or other mechanism.
375 */
376static int
377trace_seq_puts(struct trace_seq *s, const char *str)
378{
379 int len = strlen(str);
380
381 if (len > ((PAGE_SIZE - 1) - s->len))
382 return 0;
383
384 memcpy(s->buffer + s->len, str, len);
385 s->len += len;
386
387 return len;
388}
389
390static int
391trace_seq_putc(struct trace_seq *s, unsigned char c)
392{
393 if (s->len >= (PAGE_SIZE - 1))
394 return 0;
395
396 s->buffer[s->len++] = c;
397
398 return 1;
399}
400
401static int
402trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
403{
404 if (len > ((PAGE_SIZE - 1) - s->len))
405 return 0;
406
407 memcpy(s->buffer + s->len, mem, len);
408 s->len += len;
409
410 return len;
411}
412
413#define MAX_MEMHEX_BYTES 8
414#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
415
416static int
417trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
418{
419 unsigned char hex[HEX_CHARS];
420 unsigned char *data = mem;
421 int i, j;
422
423#ifdef __BIG_ENDIAN
424 for (i = 0, j = 0; i < len; i++) {
425#else
426 for (i = len-1, j = 0; i >= 0; i--) {
427#endif
428 hex[j++] = hex_asc_hi(data[i]);
429 hex[j++] = hex_asc_lo(data[i]);
430 }
431 hex[j++] = ' ';
432
433 return trace_seq_putmem(s, hex, j);
434}
435
436static int
437trace_seq_path(struct trace_seq *s, struct path *path)
438{
439 unsigned char *p;
440
441 if (s->len >= (PAGE_SIZE - 1))
442 return 0;
443 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len);
444 if (!IS_ERR(p)) {
445 p = mangle_path(s->buffer + s->len, p, "\n");
446 if (p) {
447 s->len = p - s->buffer;
448 return 1;
449 }
450 } else {
451 s->buffer[s->len++] = '?';
452 return 1;
453 }
454
455 return 0;
456}
457
458static void 330static void
459trace_seq_reset(struct trace_seq *s) 331trace_seq_reset(struct trace_seq *s)
460{ 332{
@@ -960,10 +832,10 @@ ftrace(struct trace_array *tr, struct trace_array_cpu *data,
960 trace_function(tr, data, ip, parent_ip, flags, pc); 832 trace_function(tr, data, ip, parent_ip, flags, pc);
961} 833}
962 834
963static void ftrace_trace_stack(struct trace_array *tr, 835static void __ftrace_trace_stack(struct trace_array *tr,
964 struct trace_array_cpu *data, 836 struct trace_array_cpu *data,
965 unsigned long flags, 837 unsigned long flags,
966 int skip, int pc) 838 int skip, int pc)
967{ 839{
968#ifdef CONFIG_STACKTRACE 840#ifdef CONFIG_STACKTRACE
969 struct ring_buffer_event *event; 841 struct ring_buffer_event *event;
@@ -971,9 +843,6 @@ static void ftrace_trace_stack(struct trace_array *tr,
971 struct stack_trace trace; 843 struct stack_trace trace;
972 unsigned long irq_flags; 844 unsigned long irq_flags;
973 845
974 if (!(trace_flags & TRACE_ITER_STACKTRACE))
975 return;
976
977 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 846 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
978 &irq_flags); 847 &irq_flags);
979 if (!event) 848 if (!event)
@@ -994,12 +863,23 @@ static void ftrace_trace_stack(struct trace_array *tr,
994#endif 863#endif
995} 864}
996 865
866static void ftrace_trace_stack(struct trace_array *tr,
867 struct trace_array_cpu *data,
868 unsigned long flags,
869 int skip, int pc)
870{
871 if (!(trace_flags & TRACE_ITER_STACKTRACE))
872 return;
873
874 __ftrace_trace_stack(tr, data, flags, skip, pc);
875}
876
997void __trace_stack(struct trace_array *tr, 877void __trace_stack(struct trace_array *tr,
998 struct trace_array_cpu *data, 878 struct trace_array_cpu *data,
999 unsigned long flags, 879 unsigned long flags,
1000 int skip) 880 int skip, int pc)
1001{ 881{
1002 ftrace_trace_stack(tr, data, flags, skip, preempt_count()); 882 __ftrace_trace_stack(tr, data, flags, skip, pc);
1003} 883}
1004 884
1005static void ftrace_trace_userstack(struct trace_array *tr, 885static void ftrace_trace_userstack(struct trace_array *tr,
@@ -1163,65 +1043,6 @@ ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1163 local_irq_restore(flags); 1043 local_irq_restore(flags);
1164} 1044}
1165 1045
1166#ifdef CONFIG_FUNCTION_TRACER
1167static void
1168function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
1169{
1170 struct trace_array *tr = &global_trace;
1171 struct trace_array_cpu *data;
1172 unsigned long flags;
1173 long disabled;
1174 int cpu, resched;
1175 int pc;
1176
1177 if (unlikely(!ftrace_function_enabled))
1178 return;
1179
1180 pc = preempt_count();
1181 resched = ftrace_preempt_disable();
1182 local_save_flags(flags);
1183 cpu = raw_smp_processor_id();
1184 data = tr->data[cpu];
1185 disabled = atomic_inc_return(&data->disabled);
1186
1187 if (likely(disabled == 1))
1188 trace_function(tr, data, ip, parent_ip, flags, pc);
1189
1190 atomic_dec(&data->disabled);
1191 ftrace_preempt_enable(resched);
1192}
1193
1194static void
1195function_trace_call(unsigned long ip, unsigned long parent_ip)
1196{
1197 struct trace_array *tr = &global_trace;
1198 struct trace_array_cpu *data;
1199 unsigned long flags;
1200 long disabled;
1201 int cpu;
1202 int pc;
1203
1204 if (unlikely(!ftrace_function_enabled))
1205 return;
1206
1207 /*
1208 * Need to use raw, since this must be called before the
1209 * recursive protection is performed.
1210 */
1211 local_irq_save(flags);
1212 cpu = raw_smp_processor_id();
1213 data = tr->data[cpu];
1214 disabled = atomic_inc_return(&data->disabled);
1215
1216 if (likely(disabled == 1)) {
1217 pc = preempt_count();
1218 trace_function(tr, data, ip, parent_ip, flags, pc);
1219 }
1220
1221 atomic_dec(&data->disabled);
1222 local_irq_restore(flags);
1223}
1224
1225#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1046#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1226int trace_graph_entry(struct ftrace_graph_ent *trace) 1047int trace_graph_entry(struct ftrace_graph_ent *trace)
1227{ 1048{
@@ -1279,31 +1100,6 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
1279} 1100}
1280#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 1101#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1281 1102
1282static struct ftrace_ops trace_ops __read_mostly =
1283{
1284 .func = function_trace_call,
1285};
1286
1287void tracing_start_function_trace(void)
1288{
1289 ftrace_function_enabled = 0;
1290
1291 if (trace_flags & TRACE_ITER_PREEMPTONLY)
1292 trace_ops.func = function_trace_call_preempt_only;
1293 else
1294 trace_ops.func = function_trace_call;
1295
1296 register_ftrace_function(&trace_ops);
1297 ftrace_function_enabled = 1;
1298}
1299
1300void tracing_stop_function_trace(void)
1301{
1302 ftrace_function_enabled = 0;
1303 unregister_ftrace_function(&trace_ops);
1304}
1305#endif
1306
1307enum trace_file_type { 1103enum trace_file_type {
1308 TRACE_FILE_LAT_FMT = 1, 1104 TRACE_FILE_LAT_FMT = 1,
1309 TRACE_FILE_ANNOTATE = 2, 1105 TRACE_FILE_ANNOTATE = 2,
@@ -1472,154 +1268,6 @@ static void s_stop(struct seq_file *m, void *p)
1472 mutex_unlock(&trace_types_lock); 1268 mutex_unlock(&trace_types_lock);
1473} 1269}
1474 1270
1475#ifdef CONFIG_KRETPROBES
1476static inline const char *kretprobed(const char *name)
1477{
1478 static const char tramp_name[] = "kretprobe_trampoline";
1479 int size = sizeof(tramp_name);
1480
1481 if (strncmp(tramp_name, name, size) == 0)
1482 return "[unknown/kretprobe'd]";
1483 return name;
1484}
1485#else
1486static inline const char *kretprobed(const char *name)
1487{
1488 return name;
1489}
1490#endif /* CONFIG_KRETPROBES */
1491
1492static int
1493seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1494{
1495#ifdef CONFIG_KALLSYMS
1496 char str[KSYM_SYMBOL_LEN];
1497 const char *name;
1498
1499 kallsyms_lookup(address, NULL, NULL, NULL, str);
1500
1501 name = kretprobed(str);
1502
1503 return trace_seq_printf(s, fmt, name);
1504#endif
1505 return 1;
1506}
1507
1508static int
1509seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1510 unsigned long address)
1511{
1512#ifdef CONFIG_KALLSYMS
1513 char str[KSYM_SYMBOL_LEN];
1514 const char *name;
1515
1516 sprint_symbol(str, address);
1517 name = kretprobed(str);
1518
1519 return trace_seq_printf(s, fmt, name);
1520#endif
1521 return 1;
1522}
1523
1524#ifndef CONFIG_64BIT
1525# define IP_FMT "%08lx"
1526#else
1527# define IP_FMT "%016lx"
1528#endif
1529
1530int
1531seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1532{
1533 int ret;
1534
1535 if (!ip)
1536 return trace_seq_printf(s, "0");
1537
1538 if (sym_flags & TRACE_ITER_SYM_OFFSET)
1539 ret = seq_print_sym_offset(s, "%s", ip);
1540 else
1541 ret = seq_print_sym_short(s, "%s", ip);
1542
1543 if (!ret)
1544 return 0;
1545
1546 if (sym_flags & TRACE_ITER_SYM_ADDR)
1547 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1548 return ret;
1549}
1550
1551static inline int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
1552 unsigned long ip, unsigned long sym_flags)
1553{
1554 struct file *file = NULL;
1555 unsigned long vmstart = 0;
1556 int ret = 1;
1557
1558 if (mm) {
1559 const struct vm_area_struct *vma;
1560
1561 down_read(&mm->mmap_sem);
1562 vma = find_vma(mm, ip);
1563 if (vma) {
1564 file = vma->vm_file;
1565 vmstart = vma->vm_start;
1566 }
1567 if (file) {
1568 ret = trace_seq_path(s, &file->f_path);
1569 if (ret)
1570 ret = trace_seq_printf(s, "[+0x%lx]", ip - vmstart);
1571 }
1572 up_read(&mm->mmap_sem);
1573 }
1574 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
1575 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1576 return ret;
1577}
1578
1579static int
1580seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
1581 unsigned long sym_flags)
1582{
1583 struct mm_struct *mm = NULL;
1584 int ret = 1;
1585 unsigned int i;
1586
1587 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
1588 struct task_struct *task;
1589 /*
1590 * we do the lookup on the thread group leader,
1591 * since individual threads might have already quit!
1592 */
1593 rcu_read_lock();
1594 task = find_task_by_vpid(entry->ent.tgid);
1595 if (task)
1596 mm = get_task_mm(task);
1597 rcu_read_unlock();
1598 }
1599
1600 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1601 unsigned long ip = entry->caller[i];
1602
1603 if (ip == ULONG_MAX || !ret)
1604 break;
1605 if (i && ret)
1606 ret = trace_seq_puts(s, " <- ");
1607 if (!ip) {
1608 if (ret)
1609 ret = trace_seq_puts(s, "??");
1610 continue;
1611 }
1612 if (!ret)
1613 break;
1614 if (ret)
1615 ret = seq_print_user_ip(s, mm, ip, sym_flags);
1616 }
1617
1618 if (mm)
1619 mmput(mm);
1620 return ret;
1621}
1622
1623static void print_lat_help_header(struct seq_file *m) 1271static void print_lat_help_header(struct seq_file *m)
1624{ 1272{
1625 seq_puts(m, "# _------=> CPU# \n"); 1273 seq_puts(m, "# _------=> CPU# \n");
@@ -1755,52 +1403,6 @@ lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1755 trace_seq_puts(s, " : "); 1403 trace_seq_puts(s, " : ");
1756} 1404}
1757 1405
1758static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1759
1760static int task_state_char(unsigned long state)
1761{
1762 int bit = state ? __ffs(state) + 1 : 0;
1763
1764 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
1765}
1766
1767/*
1768 * The message is supposed to contain an ending newline.
1769 * If the printing stops prematurely, try to add a newline of our own.
1770 */
1771void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1772{
1773 struct trace_entry *ent;
1774 struct trace_field_cont *cont;
1775 bool ok = true;
1776
1777 ent = peek_next_entry(iter, iter->cpu, NULL);
1778 if (!ent || ent->type != TRACE_CONT) {
1779 trace_seq_putc(s, '\n');
1780 return;
1781 }
1782
1783 do {
1784 cont = (struct trace_field_cont *)ent;
1785 if (ok)
1786 ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1787
1788 ftrace_disable_cpu();
1789
1790 if (iter->buffer_iter[iter->cpu])
1791 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1792 else
1793 ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1794
1795 ftrace_enable_cpu();
1796
1797 ent = peek_next_entry(iter, iter->cpu, NULL);
1798 } while (ent && ent->type == TRACE_CONT);
1799
1800 if (!ok)
1801 trace_seq_putc(s, '\n');
1802}
1803
1804static void test_cpu_buff_start(struct trace_iterator *iter) 1406static void test_cpu_buff_start(struct trace_iterator *iter)
1805{ 1407{
1806 struct trace_seq *s = &iter->seq; 1408 struct trace_seq *s = &iter->seq;
@@ -1824,17 +1426,14 @@ print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1824 struct trace_seq *s = &iter->seq; 1426 struct trace_seq *s = &iter->seq;
1825 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1427 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1826 struct trace_entry *next_entry; 1428 struct trace_entry *next_entry;
1429 struct trace_event *event;
1827 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE); 1430 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1828 struct trace_entry *entry = iter->ent; 1431 struct trace_entry *entry = iter->ent;
1829 unsigned long abs_usecs; 1432 unsigned long abs_usecs;
1830 unsigned long rel_usecs; 1433 unsigned long rel_usecs;
1831 u64 next_ts; 1434 u64 next_ts;
1832 char *comm; 1435 char *comm;
1833 int S, T; 1436 int ret;
1834 int i;
1835
1836 if (entry->type == TRACE_CONT)
1837 return TRACE_TYPE_HANDLED;
1838 1437
1839 test_cpu_buff_start(iter); 1438 test_cpu_buff_start(iter);
1840 1439
@@ -1859,96 +1458,16 @@ print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1859 lat_print_generic(s, entry, cpu); 1458 lat_print_generic(s, entry, cpu);
1860 lat_print_timestamp(s, abs_usecs, rel_usecs); 1459 lat_print_timestamp(s, abs_usecs, rel_usecs);
1861 } 1460 }
1862 switch (entry->type) {
1863 case TRACE_FN: {
1864 struct ftrace_entry *field;
1865 1461
1866 trace_assign_type(field, entry); 1462 event = ftrace_find_event(entry->type);
1867 1463 if (event && event->latency_trace) {
1868 seq_print_ip_sym(s, field->ip, sym_flags); 1464 ret = event->latency_trace(s, entry, sym_flags);
1869 trace_seq_puts(s, " ("); 1465 if (ret)
1870 seq_print_ip_sym(s, field->parent_ip, sym_flags); 1466 return ret;
1871 trace_seq_puts(s, ")\n"); 1467 return TRACE_TYPE_HANDLED;
1872 break;
1873 }
1874 case TRACE_CTX:
1875 case TRACE_WAKE: {
1876 struct ctx_switch_entry *field;
1877
1878 trace_assign_type(field, entry);
1879
1880 T = task_state_char(field->next_state);
1881 S = task_state_char(field->prev_state);
1882 comm = trace_find_cmdline(field->next_pid);
1883 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1884 field->prev_pid,
1885 field->prev_prio,
1886 S, entry->type == TRACE_CTX ? "==>" : " +",
1887 field->next_cpu,
1888 field->next_pid,
1889 field->next_prio,
1890 T, comm);
1891 break;
1892 }
1893 case TRACE_SPECIAL: {
1894 struct special_entry *field;
1895
1896 trace_assign_type(field, entry);
1897
1898 trace_seq_printf(s, "# %ld %ld %ld\n",
1899 field->arg1,
1900 field->arg2,
1901 field->arg3);
1902 break;
1903 }
1904 case TRACE_STACK: {
1905 struct stack_entry *field;
1906
1907 trace_assign_type(field, entry);
1908
1909 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1910 if (i)
1911 trace_seq_puts(s, " <= ");
1912 seq_print_ip_sym(s, field->caller[i], sym_flags);
1913 }
1914 trace_seq_puts(s, "\n");
1915 break;
1916 }
1917 case TRACE_PRINT: {
1918 struct print_entry *field;
1919
1920 trace_assign_type(field, entry);
1921
1922 seq_print_ip_sym(s, field->ip, sym_flags);
1923 trace_seq_printf(s, ": %s", field->buf);
1924 if (entry->flags & TRACE_FLAG_CONT)
1925 trace_seq_print_cont(s, iter);
1926 break;
1927 }
1928 case TRACE_BRANCH: {
1929 struct trace_branch *field;
1930
1931 trace_assign_type(field, entry);
1932
1933 trace_seq_printf(s, "[%s] %s:%s:%d\n",
1934 field->correct ? " ok " : " MISS ",
1935 field->func,
1936 field->file,
1937 field->line);
1938 break;
1939 } 1468 }
1940 case TRACE_USER_STACK: {
1941 struct userstack_entry *field;
1942 1469
1943 trace_assign_type(field, entry); 1470 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1944
1945 seq_print_userip_objs(field, s, sym_flags);
1946 trace_seq_putc(s, '\n');
1947 break;
1948 }
1949 default:
1950 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1951 }
1952 return TRACE_TYPE_HANDLED; 1471 return TRACE_TYPE_HANDLED;
1953} 1472}
1954 1473
@@ -1957,19 +1476,15 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1957 struct trace_seq *s = &iter->seq; 1476 struct trace_seq *s = &iter->seq;
1958 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1477 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1959 struct trace_entry *entry; 1478 struct trace_entry *entry;
1479 struct trace_event *event;
1960 unsigned long usec_rem; 1480 unsigned long usec_rem;
1961 unsigned long long t; 1481 unsigned long long t;
1962 unsigned long secs; 1482 unsigned long secs;
1963 char *comm; 1483 char *comm;
1964 int ret; 1484 int ret;
1965 int S, T;
1966 int i;
1967 1485
1968 entry = iter->ent; 1486 entry = iter->ent;
1969 1487
1970 if (entry->type == TRACE_CONT)
1971 return TRACE_TYPE_HANDLED;
1972
1973 test_cpu_buff_start(iter); 1488 test_cpu_buff_start(iter);
1974 1489
1975 comm = trace_find_cmdline(iter->ent->pid); 1490 comm = trace_find_cmdline(iter->ent->pid);
@@ -1988,129 +1503,17 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1988 if (!ret) 1503 if (!ret)
1989 return TRACE_TYPE_PARTIAL_LINE; 1504 return TRACE_TYPE_PARTIAL_LINE;
1990 1505
1991 switch (entry->type) { 1506 event = ftrace_find_event(entry->type);
1992 case TRACE_FN: { 1507 if (event && event->trace) {
1993 struct ftrace_entry *field; 1508 ret = event->trace(s, entry, sym_flags);
1994 1509 if (ret)
1995 trace_assign_type(field, entry); 1510 return ret;
1996 1511 return TRACE_TYPE_HANDLED;
1997 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1998 if (!ret)
1999 return TRACE_TYPE_PARTIAL_LINE;
2000 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
2001 field->parent_ip) {
2002 ret = trace_seq_printf(s, " <-");
2003 if (!ret)
2004 return TRACE_TYPE_PARTIAL_LINE;
2005 ret = seq_print_ip_sym(s,
2006 field->parent_ip,
2007 sym_flags);
2008 if (!ret)
2009 return TRACE_TYPE_PARTIAL_LINE;
2010 }
2011 ret = trace_seq_printf(s, "\n");
2012 if (!ret)
2013 return TRACE_TYPE_PARTIAL_LINE;
2014 break;
2015 }
2016 case TRACE_CTX:
2017 case TRACE_WAKE: {
2018 struct ctx_switch_entry *field;
2019
2020 trace_assign_type(field, entry);
2021
2022 T = task_state_char(field->next_state);
2023 S = task_state_char(field->prev_state);
2024 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
2025 field->prev_pid,
2026 field->prev_prio,
2027 S,
2028 entry->type == TRACE_CTX ? "==>" : " +",
2029 field->next_cpu,
2030 field->next_pid,
2031 field->next_prio,
2032 T);
2033 if (!ret)
2034 return TRACE_TYPE_PARTIAL_LINE;
2035 break;
2036 }
2037 case TRACE_SPECIAL: {
2038 struct special_entry *field;
2039
2040 trace_assign_type(field, entry);
2041
2042 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
2043 field->arg1,
2044 field->arg2,
2045 field->arg3);
2046 if (!ret)
2047 return TRACE_TYPE_PARTIAL_LINE;
2048 break;
2049 }
2050 case TRACE_STACK: {
2051 struct stack_entry *field;
2052
2053 trace_assign_type(field, entry);
2054
2055 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
2056 if (i) {
2057 ret = trace_seq_puts(s, " <= ");
2058 if (!ret)
2059 return TRACE_TYPE_PARTIAL_LINE;
2060 }
2061 ret = seq_print_ip_sym(s, field->caller[i],
2062 sym_flags);
2063 if (!ret)
2064 return TRACE_TYPE_PARTIAL_LINE;
2065 }
2066 ret = trace_seq_puts(s, "\n");
2067 if (!ret)
2068 return TRACE_TYPE_PARTIAL_LINE;
2069 break;
2070 }
2071 case TRACE_PRINT: {
2072 struct print_entry *field;
2073
2074 trace_assign_type(field, entry);
2075
2076 seq_print_ip_sym(s, field->ip, sym_flags);
2077 trace_seq_printf(s, ": %s", field->buf);
2078 if (entry->flags & TRACE_FLAG_CONT)
2079 trace_seq_print_cont(s, iter);
2080 break;
2081 }
2082 case TRACE_GRAPH_RET: {
2083 return print_graph_function(iter);
2084 }
2085 case TRACE_GRAPH_ENT: {
2086 return print_graph_function(iter);
2087 }
2088 case TRACE_BRANCH: {
2089 struct trace_branch *field;
2090
2091 trace_assign_type(field, entry);
2092
2093 trace_seq_printf(s, "[%s] %s:%s:%d\n",
2094 field->correct ? " ok " : " MISS ",
2095 field->func,
2096 field->file,
2097 field->line);
2098 break;
2099 } 1512 }
2100 case TRACE_USER_STACK: { 1513 ret = trace_seq_printf(s, "Unknown type %d\n", entry->type);
2101 struct userstack_entry *field; 1514 if (!ret)
2102 1515 return TRACE_TYPE_PARTIAL_LINE;
2103 trace_assign_type(field, entry);
2104 1516
2105 ret = seq_print_userip_objs(field, s, sym_flags);
2106 if (!ret)
2107 return TRACE_TYPE_PARTIAL_LINE;
2108 ret = trace_seq_putc(s, '\n');
2109 if (!ret)
2110 return TRACE_TYPE_PARTIAL_LINE;
2111 break;
2112 }
2113 }
2114 return TRACE_TYPE_HANDLED; 1517 return TRACE_TYPE_HANDLED;
2115} 1518}
2116 1519
@@ -2118,152 +1521,47 @@ static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2118{ 1521{
2119 struct trace_seq *s = &iter->seq; 1522 struct trace_seq *s = &iter->seq;
2120 struct trace_entry *entry; 1523 struct trace_entry *entry;
1524 struct trace_event *event;
2121 int ret; 1525 int ret;
2122 int S, T;
2123 1526
2124 entry = iter->ent; 1527 entry = iter->ent;
2125 1528
2126 if (entry->type == TRACE_CONT)
2127 return TRACE_TYPE_HANDLED;
2128
2129 ret = trace_seq_printf(s, "%d %d %llu ", 1529 ret = trace_seq_printf(s, "%d %d %llu ",
2130 entry->pid, iter->cpu, iter->ts); 1530 entry->pid, iter->cpu, iter->ts);
2131 if (!ret) 1531 if (!ret)
2132 return TRACE_TYPE_PARTIAL_LINE; 1532 return TRACE_TYPE_PARTIAL_LINE;
2133 1533
2134 switch (entry->type) { 1534 event = ftrace_find_event(entry->type);
2135 case TRACE_FN: { 1535 if (event && event->raw) {
2136 struct ftrace_entry *field; 1536 ret = event->raw(s, entry, 0);
2137 1537 if (ret)
2138 trace_assign_type(field, entry); 1538 return ret;
2139 1539 return TRACE_TYPE_HANDLED;
2140 ret = trace_seq_printf(s, "%x %x\n",
2141 field->ip,
2142 field->parent_ip);
2143 if (!ret)
2144 return TRACE_TYPE_PARTIAL_LINE;
2145 break;
2146 }
2147 case TRACE_CTX:
2148 case TRACE_WAKE: {
2149 struct ctx_switch_entry *field;
2150
2151 trace_assign_type(field, entry);
2152
2153 T = task_state_char(field->next_state);
2154 S = entry->type == TRACE_WAKE ? '+' :
2155 task_state_char(field->prev_state);
2156 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
2157 field->prev_pid,
2158 field->prev_prio,
2159 S,
2160 field->next_cpu,
2161 field->next_pid,
2162 field->next_prio,
2163 T);
2164 if (!ret)
2165 return TRACE_TYPE_PARTIAL_LINE;
2166 break;
2167 }
2168 case TRACE_SPECIAL:
2169 case TRACE_USER_STACK:
2170 case TRACE_STACK: {
2171 struct special_entry *field;
2172
2173 trace_assign_type(field, entry);
2174
2175 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
2176 field->arg1,
2177 field->arg2,
2178 field->arg3);
2179 if (!ret)
2180 return TRACE_TYPE_PARTIAL_LINE;
2181 break;
2182 } 1540 }
2183 case TRACE_PRINT: { 1541 ret = trace_seq_printf(s, "%d ?\n", entry->type);
2184 struct print_entry *field; 1542 if (!ret)
2185 1543 return TRACE_TYPE_PARTIAL_LINE;
2186 trace_assign_type(field, entry);
2187 1544
2188 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
2189 if (entry->flags & TRACE_FLAG_CONT)
2190 trace_seq_print_cont(s, iter);
2191 break;
2192 }
2193 }
2194 return TRACE_TYPE_HANDLED; 1545 return TRACE_TYPE_HANDLED;
2195} 1546}
2196 1547
2197#define SEQ_PUT_FIELD_RET(s, x) \
2198do { \
2199 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
2200 return 0; \
2201} while (0)
2202
2203#define SEQ_PUT_HEX_FIELD_RET(s, x) \
2204do { \
2205 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
2206 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
2207 return 0; \
2208} while (0)
2209
2210static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 1548static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2211{ 1549{
2212 struct trace_seq *s = &iter->seq; 1550 struct trace_seq *s = &iter->seq;
2213 unsigned char newline = '\n'; 1551 unsigned char newline = '\n';
2214 struct trace_entry *entry; 1552 struct trace_entry *entry;
2215 int S, T; 1553 struct trace_event *event;
2216 1554
2217 entry = iter->ent; 1555 entry = iter->ent;
2218 1556
2219 if (entry->type == TRACE_CONT)
2220 return TRACE_TYPE_HANDLED;
2221
2222 SEQ_PUT_HEX_FIELD_RET(s, entry->pid); 1557 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2223 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu); 1558 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2224 SEQ_PUT_HEX_FIELD_RET(s, iter->ts); 1559 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2225 1560
2226 switch (entry->type) { 1561 event = ftrace_find_event(entry->type);
2227 case TRACE_FN: { 1562 if (event && event->hex)
2228 struct ftrace_entry *field; 1563 event->hex(s, entry, 0);
2229
2230 trace_assign_type(field, entry);
2231
2232 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
2233 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
2234 break;
2235 }
2236 case TRACE_CTX:
2237 case TRACE_WAKE: {
2238 struct ctx_switch_entry *field;
2239
2240 trace_assign_type(field, entry);
2241
2242 T = task_state_char(field->next_state);
2243 S = entry->type == TRACE_WAKE ? '+' :
2244 task_state_char(field->prev_state);
2245 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
2246 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
2247 SEQ_PUT_HEX_FIELD_RET(s, S);
2248 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
2249 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
2250 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
2251 SEQ_PUT_HEX_FIELD_RET(s, T);
2252 break;
2253 }
2254 case TRACE_SPECIAL:
2255 case TRACE_USER_STACK:
2256 case TRACE_STACK: {
2257 struct special_entry *field;
2258
2259 trace_assign_type(field, entry);
2260 1564
2261 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
2262 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
2263 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
2264 break;
2265 }
2266 }
2267 SEQ_PUT_FIELD_RET(s, newline); 1565 SEQ_PUT_FIELD_RET(s, newline);
2268 1566
2269 return TRACE_TYPE_HANDLED; 1567 return TRACE_TYPE_HANDLED;
@@ -2282,9 +1580,6 @@ static enum print_line_t print_printk_msg_only(struct trace_iterator *iter)
2282 if (!ret) 1580 if (!ret)
2283 return TRACE_TYPE_PARTIAL_LINE; 1581 return TRACE_TYPE_PARTIAL_LINE;
2284 1582
2285 if (entry->flags & TRACE_FLAG_CONT)
2286 trace_seq_print_cont(s, iter);
2287
2288 return TRACE_TYPE_HANDLED; 1583 return TRACE_TYPE_HANDLED;
2289} 1584}
2290 1585
@@ -2292,53 +1587,19 @@ static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2292{ 1587{
2293 struct trace_seq *s = &iter->seq; 1588 struct trace_seq *s = &iter->seq;
2294 struct trace_entry *entry; 1589 struct trace_entry *entry;
1590 struct trace_event *event;
2295 1591
2296 entry = iter->ent; 1592 entry = iter->ent;
2297 1593
2298 if (entry->type == TRACE_CONT)
2299 return TRACE_TYPE_HANDLED;
2300
2301 SEQ_PUT_FIELD_RET(s, entry->pid); 1594 SEQ_PUT_FIELD_RET(s, entry->pid);
2302 SEQ_PUT_FIELD_RET(s, entry->cpu); 1595 SEQ_PUT_FIELD_RET(s, entry->cpu);
2303 SEQ_PUT_FIELD_RET(s, iter->ts); 1596 SEQ_PUT_FIELD_RET(s, iter->ts);
2304 1597
2305 switch (entry->type) { 1598 event = ftrace_find_event(entry->type);
2306 case TRACE_FN: { 1599 if (event && event->binary)
2307 struct ftrace_entry *field; 1600 event->binary(s, entry, 0);
2308
2309 trace_assign_type(field, entry);
2310
2311 SEQ_PUT_FIELD_RET(s, field->ip);
2312 SEQ_PUT_FIELD_RET(s, field->parent_ip);
2313 break;
2314 }
2315 case TRACE_CTX: {
2316 struct ctx_switch_entry *field;
2317
2318 trace_assign_type(field, entry);
2319 1601
2320 SEQ_PUT_FIELD_RET(s, field->prev_pid); 1602 return TRACE_TYPE_HANDLED;
2321 SEQ_PUT_FIELD_RET(s, field->prev_prio);
2322 SEQ_PUT_FIELD_RET(s, field->prev_state);
2323 SEQ_PUT_FIELD_RET(s, field->next_pid);
2324 SEQ_PUT_FIELD_RET(s, field->next_prio);
2325 SEQ_PUT_FIELD_RET(s, field->next_state);
2326 break;
2327 }
2328 case TRACE_SPECIAL:
2329 case TRACE_USER_STACK:
2330 case TRACE_STACK: {
2331 struct special_entry *field;
2332
2333 trace_assign_type(field, entry);
2334
2335 SEQ_PUT_FIELD_RET(s, field->arg1);
2336 SEQ_PUT_FIELD_RET(s, field->arg2);
2337 SEQ_PUT_FIELD_RET(s, field->arg3);
2338 break;
2339 }
2340 }
2341 return 1;
2342} 1603}
2343 1604
2344static int trace_empty(struct trace_iterator *iter) 1605static int trace_empty(struct trace_iterator *iter)
@@ -3736,7 +2997,7 @@ static struct notifier_block trace_die_notifier = {
3736 * it if we decide to change what log level the ftrace dump 2997 * it if we decide to change what log level the ftrace dump
3737 * should be at. 2998 * should be at.
3738 */ 2999 */
3739#define KERN_TRACE KERN_INFO 3000#define KERN_TRACE KERN_EMERG
3740 3001
3741static void 3002static void
3742trace_printk_seq(struct trace_seq *s) 3003trace_printk_seq(struct trace_seq *s)
@@ -3770,6 +3031,7 @@ void ftrace_dump(void)
3770 dump_ran = 1; 3031 dump_ran = 1;
3771 3032
3772 /* No turning back! */ 3033 /* No turning back! */
3034 tracing_off();
3773 ftrace_kill(); 3035 ftrace_kill();
3774 3036
3775 for_each_tracing_cpu(cpu) { 3037 for_each_tracing_cpu(cpu) {
@@ -3877,7 +3139,6 @@ __init static int tracer_alloc_buffers(void)
3877#else 3139#else
3878 current_trace = &nop_trace; 3140 current_trace = &nop_trace;
3879#endif 3141#endif
3880
3881 /* All seems OK, enable tracing */ 3142 /* All seems OK, enable tracing */
3882 tracing_disabled = 0; 3143 tracing_disabled = 0;
3883 3144
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 4d3d381bfd95..54b72781e920 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -9,6 +9,7 @@
9#include <linux/mmiotrace.h> 9#include <linux/mmiotrace.h>
10#include <linux/ftrace.h> 10#include <linux/ftrace.h>
11#include <trace/boot.h> 11#include <trace/boot.h>
12#include <trace/kmemtrace.h>
12 13
13enum trace_type { 14enum trace_type {
14 __TRACE_FIRST_TYPE = 0, 15 __TRACE_FIRST_TYPE = 0,
@@ -16,7 +17,6 @@ enum trace_type {
16 TRACE_FN, 17 TRACE_FN,
17 TRACE_CTX, 18 TRACE_CTX,
18 TRACE_WAKE, 19 TRACE_WAKE,
19 TRACE_CONT,
20 TRACE_STACK, 20 TRACE_STACK,
21 TRACE_PRINT, 21 TRACE_PRINT,
22 TRACE_SPECIAL, 22 TRACE_SPECIAL,
@@ -29,9 +29,11 @@ enum trace_type {
29 TRACE_GRAPH_ENT, 29 TRACE_GRAPH_ENT,
30 TRACE_USER_STACK, 30 TRACE_USER_STACK,
31 TRACE_HW_BRANCHES, 31 TRACE_HW_BRANCHES,
32 TRACE_KMEM_ALLOC,
33 TRACE_KMEM_FREE,
32 TRACE_POWER, 34 TRACE_POWER,
33 35
34 __TRACE_LAST_TYPE 36 __TRACE_LAST_TYPE,
35}; 37};
36 38
37/* 39/*
@@ -170,6 +172,24 @@ struct trace_power {
170 struct power_trace state_data; 172 struct power_trace state_data;
171}; 173};
172 174
175struct kmemtrace_alloc_entry {
176 struct trace_entry ent;
177 enum kmemtrace_type_id type_id;
178 unsigned long call_site;
179 const void *ptr;
180 size_t bytes_req;
181 size_t bytes_alloc;
182 gfp_t gfp_flags;
183 int node;
184};
185
186struct kmemtrace_free_entry {
187 struct trace_entry ent;
188 enum kmemtrace_type_id type_id;
189 unsigned long call_site;
190 const void *ptr;
191};
192
173/* 193/*
174 * trace_flag_type is an enumeration that holds different 194 * trace_flag_type is an enumeration that holds different
175 * states when a trace occurs. These are: 195 * states when a trace occurs. These are:
@@ -178,7 +198,6 @@ struct trace_power {
178 * NEED_RESCED - reschedule is requested 198 * NEED_RESCED - reschedule is requested
179 * HARDIRQ - inside an interrupt handler 199 * HARDIRQ - inside an interrupt handler
180 * SOFTIRQ - inside a softirq handler 200 * SOFTIRQ - inside a softirq handler
181 * CONT - multiple entries hold the trace item
182 */ 201 */
183enum trace_flag_type { 202enum trace_flag_type {
184 TRACE_FLAG_IRQS_OFF = 0x01, 203 TRACE_FLAG_IRQS_OFF = 0x01,
@@ -186,7 +205,6 @@ enum trace_flag_type {
186 TRACE_FLAG_NEED_RESCHED = 0x04, 205 TRACE_FLAG_NEED_RESCHED = 0x04,
187 TRACE_FLAG_HARDIRQ = 0x08, 206 TRACE_FLAG_HARDIRQ = 0x08,
188 TRACE_FLAG_SOFTIRQ = 0x10, 207 TRACE_FLAG_SOFTIRQ = 0x10,
189 TRACE_FLAG_CONT = 0x20,
190}; 208};
191 209
192#define TRACE_BUF_SIZE 1024 210#define TRACE_BUF_SIZE 1024
@@ -262,7 +280,6 @@ extern void __ftrace_bad_type(void);
262 do { \ 280 do { \
263 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \ 281 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \
264 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \ 282 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \
265 IF_ASSIGN(var, ent, struct trace_field_cont, TRACE_CONT); \
266 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \ 283 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \
267 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\ 284 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
268 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \ 285 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \
@@ -280,6 +297,10 @@ extern void __ftrace_bad_type(void);
280 TRACE_GRAPH_RET); \ 297 TRACE_GRAPH_RET); \
281 IF_ASSIGN(var, ent, struct hw_branch_entry, TRACE_HW_BRANCHES);\ 298 IF_ASSIGN(var, ent, struct hw_branch_entry, TRACE_HW_BRANCHES);\
282 IF_ASSIGN(var, ent, struct trace_power, TRACE_POWER); \ 299 IF_ASSIGN(var, ent, struct trace_power, TRACE_POWER); \
300 IF_ASSIGN(var, ent, struct kmemtrace_alloc_entry, \
301 TRACE_KMEM_ALLOC); \
302 IF_ASSIGN(var, ent, struct kmemtrace_free_entry, \
303 TRACE_KMEM_FREE); \
283 __ftrace_bad_type(); \ 304 __ftrace_bad_type(); \
284 } while (0) 305 } while (0)
285 306
@@ -313,6 +334,7 @@ struct tracer_flags {
313/* Makes more easy to define a tracer opt */ 334/* Makes more easy to define a tracer opt */
314#define TRACER_OPT(s, b) .name = #s, .bit = b 335#define TRACER_OPT(s, b) .name = #s, .bit = b
315 336
337
316/* 338/*
317 * A specific tracer, represented by methods that operate on a trace array: 339 * A specific tracer, represented by methods that operate on a trace array:
318 */ 340 */
@@ -340,6 +362,7 @@ struct tracer {
340 struct tracer *next; 362 struct tracer *next;
341 int print_max; 363 int print_max;
342 struct tracer_flags *flags; 364 struct tracer_flags *flags;
365 struct tracer_stat *stats;
343}; 366};
344 367
345struct trace_seq { 368struct trace_seq {
@@ -434,15 +457,12 @@ void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
434void update_max_tr_single(struct trace_array *tr, 457void update_max_tr_single(struct trace_array *tr,
435 struct task_struct *tsk, int cpu); 458 struct task_struct *tsk, int cpu);
436 459
437extern cycle_t ftrace_now(int cpu); 460void __trace_stack(struct trace_array *tr,
461 struct trace_array_cpu *data,
462 unsigned long flags,
463 int skip, int pc);
438 464
439#ifdef CONFIG_FUNCTION_TRACER 465extern cycle_t ftrace_now(int cpu);
440void tracing_start_function_trace(void);
441void tracing_stop_function_trace(void);
442#else
443# define tracing_start_function_trace() do { } while (0)
444# define tracing_stop_function_trace() do { } while (0)
445#endif
446 466
447#ifdef CONFIG_CONTEXT_SWITCH_TRACER 467#ifdef CONFIG_CONTEXT_SWITCH_TRACER
448typedef void 468typedef void
@@ -456,10 +476,10 @@ struct tracer_switch_ops {
456 void *private; 476 void *private;
457 struct tracer_switch_ops *next; 477 struct tracer_switch_ops *next;
458}; 478};
459
460char *trace_find_cmdline(int pid);
461#endif /* CONFIG_CONTEXT_SWITCH_TRACER */ 479#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
462 480
481extern char *trace_find_cmdline(int pid);
482
463#ifdef CONFIG_DYNAMIC_FTRACE 483#ifdef CONFIG_DYNAMIC_FTRACE
464extern unsigned long ftrace_update_tot_cnt; 484extern unsigned long ftrace_update_tot_cnt;
465#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func 485#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
@@ -488,15 +508,6 @@ extern int trace_selftest_startup_branch(struct tracer *trace,
488#endif /* CONFIG_FTRACE_STARTUP_TEST */ 508#endif /* CONFIG_FTRACE_STARTUP_TEST */
489 509
490extern void *head_page(struct trace_array_cpu *data); 510extern void *head_page(struct trace_array_cpu *data);
491extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...);
492extern void trace_seq_print_cont(struct trace_seq *s,
493 struct trace_iterator *iter);
494
495extern int
496seq_print_ip_sym(struct trace_seq *s, unsigned long ip,
497 unsigned long sym_flags);
498extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
499 size_t cnt);
500extern long ns2usecs(cycle_t nsec); 511extern long ns2usecs(cycle_t nsec);
501extern int 512extern int
502trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args); 513trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args);
diff --git a/kernel/trace/trace_boot.c b/kernel/trace/trace_boot.c
index 366c8c333e13..0e94b3d091f7 100644
--- a/kernel/trace/trace_boot.c
+++ b/kernel/trace/trace_boot.c
@@ -11,6 +11,7 @@
11#include <linux/kallsyms.h> 11#include <linux/kallsyms.h>
12 12
13#include "trace.h" 13#include "trace.h"
14#include "trace_output.h"
14 15
15static struct trace_array *boot_trace; 16static struct trace_array *boot_trace;
16static bool pre_initcalls_finished; 17static bool pre_initcalls_finished;
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 6c00feb3bac7..ca017e0a9a27 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -14,12 +14,17 @@
14#include <linux/hash.h> 14#include <linux/hash.h>
15#include <linux/fs.h> 15#include <linux/fs.h>
16#include <asm/local.h> 16#include <asm/local.h>
17
17#include "trace.h" 18#include "trace.h"
19#include "trace_stat.h"
20#include "trace_output.h"
18 21
19#ifdef CONFIG_BRANCH_TRACER 22#ifdef CONFIG_BRANCH_TRACER
20 23
24static struct tracer branch_trace;
21static int branch_tracing_enabled __read_mostly; 25static int branch_tracing_enabled __read_mostly;
22static DEFINE_MUTEX(branch_tracing_mutex); 26static DEFINE_MUTEX(branch_tracing_mutex);
27
23static struct trace_array *branch_tracer; 28static struct trace_array *branch_tracer;
24 29
25static void 30static void
@@ -142,22 +147,74 @@ static void branch_trace_reset(struct trace_array *tr)
142 stop_branch_trace(tr); 147 stop_branch_trace(tr);
143} 148}
144 149
145struct tracer branch_trace __read_mostly = 150static int
151trace_print_print(struct trace_seq *s, struct trace_entry *entry, int flags)
152{
153 struct print_entry *field;
154
155 trace_assign_type(field, entry);
156
157 if (seq_print_ip_sym(s, field->ip, flags))
158 goto partial;
159
160 if (trace_seq_printf(s, ": %s", field->buf))
161 goto partial;
162
163 partial:
164 return TRACE_TYPE_PARTIAL_LINE;
165}
166
167static int
168trace_branch_print(struct trace_seq *s, struct trace_entry *entry, int flags)
169{
170 struct trace_branch *field;
171
172 trace_assign_type(field, entry);
173
174 if (trace_seq_printf(s, "[%s] %s:%s:%d\n",
175 field->correct ? " ok " : " MISS ",
176 field->func,
177 field->file,
178 field->line))
179 return TRACE_TYPE_PARTIAL_LINE;
180
181 return 0;
182}
183
184
185static struct trace_event trace_branch_event = {
186 .type = TRACE_BRANCH,
187 .trace = trace_branch_print,
188 .latency_trace = trace_branch_print,
189 .raw = trace_nop_print,
190 .hex = trace_nop_print,
191 .binary = trace_nop_print,
192};
193
194static struct tracer branch_trace __read_mostly =
146{ 195{
147 .name = "branch", 196 .name = "branch",
148 .init = branch_trace_init, 197 .init = branch_trace_init,
149 .reset = branch_trace_reset, 198 .reset = branch_trace_reset,
150#ifdef CONFIG_FTRACE_SELFTEST 199#ifdef CONFIG_FTRACE_SELFTEST
151 .selftest = trace_selftest_startup_branch, 200 .selftest = trace_selftest_startup_branch,
152#endif 201#endif /* CONFIG_FTRACE_SELFTEST */
153}; 202};
154 203
155__init static int init_branch_trace(void) 204__init static int init_branch_tracer(void)
156{ 205{
206 int ret;
207
208 ret = register_ftrace_event(&trace_branch_event);
209 if (!ret) {
210 printk(KERN_WARNING "Warning: could not register "
211 "branch events\n");
212 return 1;
213 }
157 return register_tracer(&branch_trace); 214 return register_tracer(&branch_trace);
158} 215}
216device_initcall(init_branch_tracer);
159 217
160device_initcall(init_branch_trace);
161#else 218#else
162static inline 219static inline
163void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) 220void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
@@ -183,66 +240,39 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect)
183} 240}
184EXPORT_SYMBOL(ftrace_likely_update); 241EXPORT_SYMBOL(ftrace_likely_update);
185 242
186struct ftrace_pointer { 243extern unsigned long __start_annotated_branch_profile[];
187 void *start; 244extern unsigned long __stop_annotated_branch_profile[];
188 void *stop;
189 int hit;
190};
191 245
192static void * 246static int annotated_branch_stat_headers(struct seq_file *m)
193t_next(struct seq_file *m, void *v, loff_t *pos)
194{ 247{
195 const struct ftrace_pointer *f = m->private; 248 seq_printf(m, " correct incorrect %% ");
196 struct ftrace_branch_data *p = v; 249 seq_printf(m, " Function "
197 250 " File Line\n"
198 (*pos)++; 251 " ------- --------- - "
199 252 " -------- "
200 if (v == (void *)1) 253 " ---- ----\n");
201 return f->start; 254 return 0;
202
203 ++p;
204
205 if ((void *)p >= (void *)f->stop)
206 return NULL;
207
208 return p;
209} 255}
210 256
211static void *t_start(struct seq_file *m, loff_t *pos) 257static inline long get_incorrect_percent(struct ftrace_branch_data *p)
212{ 258{
213 void *t = (void *)1; 259 long percent;
214 loff_t l = 0;
215
216 for (; t && l < *pos; t = t_next(m, t, &l))
217 ;
218 260
219 return t; 261 if (p->correct) {
220} 262 percent = p->incorrect * 100;
263 percent /= p->correct + p->incorrect;
264 } else
265 percent = p->incorrect ? 100 : -1;
221 266
222static void t_stop(struct seq_file *m, void *p) 267 return percent;
223{
224} 268}
225 269
226static int t_show(struct seq_file *m, void *v) 270static int branch_stat_show(struct seq_file *m, void *v)
227{ 271{
228 const struct ftrace_pointer *fp = m->private;
229 struct ftrace_branch_data *p = v; 272 struct ftrace_branch_data *p = v;
230 const char *f; 273 const char *f;
231 long percent; 274 long percent;
232 275
233 if (v == (void *)1) {
234 if (fp->hit)
235 seq_printf(m, " miss hit %% ");
236 else
237 seq_printf(m, " correct incorrect %% ");
238 seq_printf(m, " Function "
239 " File Line\n"
240 " ------- --------- - "
241 " -------- "
242 " ---- ----\n");
243 return 0;
244 }
245
246 /* Only print the file, not the path */ 276 /* Only print the file, not the path */
247 f = p->file + strlen(p->file); 277 f = p->file + strlen(p->file);
248 while (f >= p->file && *f != '/') 278 while (f >= p->file && *f != '/')
@@ -252,11 +282,7 @@ static int t_show(struct seq_file *m, void *v)
252 /* 282 /*
253 * The miss is overlayed on correct, and hit on incorrect. 283 * The miss is overlayed on correct, and hit on incorrect.
254 */ 284 */
255 if (p->correct) { 285 percent = get_incorrect_percent(p);
256 percent = p->incorrect * 100;
257 percent /= p->correct + p->incorrect;
258 } else
259 percent = p->incorrect ? 100 : -1;
260 286
261 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); 287 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect);
262 if (percent < 0) 288 if (percent < 0)
@@ -267,76 +293,118 @@ static int t_show(struct seq_file *m, void *v)
267 return 0; 293 return 0;
268} 294}
269 295
270static struct seq_operations tracing_likely_seq_ops = { 296static void *annotated_branch_stat_start(void)
271 .start = t_start, 297{
272 .next = t_next, 298 return __start_annotated_branch_profile;
273 .stop = t_stop, 299}
274 .show = t_show, 300
301static void *
302annotated_branch_stat_next(void *v, int idx)
303{
304 struct ftrace_branch_data *p = v;
305
306 ++p;
307
308 if ((void *)p >= (void *)__stop_annotated_branch_profile)
309 return NULL;
310
311 return p;
312}
313
314static int annotated_branch_stat_cmp(void *p1, void *p2)
315{
316 struct ftrace_branch_data *a = p1;
317 struct ftrace_branch_data *b = p2;
318
319 long percent_a, percent_b;
320
321 percent_a = get_incorrect_percent(a);
322 percent_b = get_incorrect_percent(b);
323
324 if (percent_a < percent_b)
325 return -1;
326 if (percent_a > percent_b)
327 return 1;
328 else
329 return 0;
330}
331
332static struct tracer_stat annotated_branch_stats = {
333 .name = "branch_annotated",
334 .stat_start = annotated_branch_stat_start,
335 .stat_next = annotated_branch_stat_next,
336 .stat_cmp = annotated_branch_stat_cmp,
337 .stat_headers = annotated_branch_stat_headers,
338 .stat_show = branch_stat_show
275}; 339};
276 340
277static int tracing_branch_open(struct inode *inode, struct file *file) 341__init static int init_annotated_branch_stats(void)
278{ 342{
279 int ret; 343 int ret;
280 344
281 ret = seq_open(file, &tracing_likely_seq_ops); 345 ret = register_stat_tracer(&annotated_branch_stats);
282 if (!ret) { 346 if (!ret) {
283 struct seq_file *m = file->private_data; 347 printk(KERN_WARNING "Warning: could not register "
284 m->private = (void *)inode->i_private; 348 "annotated branches stats\n");
349 return 1;
285 } 350 }
286 351 return 0;
287 return ret;
288} 352}
289 353fs_initcall(init_annotated_branch_stats);
290static const struct file_operations tracing_branch_fops = {
291 .open = tracing_branch_open,
292 .read = seq_read,
293 .llseek = seq_lseek,
294};
295 354
296#ifdef CONFIG_PROFILE_ALL_BRANCHES 355#ifdef CONFIG_PROFILE_ALL_BRANCHES
356
297extern unsigned long __start_branch_profile[]; 357extern unsigned long __start_branch_profile[];
298extern unsigned long __stop_branch_profile[]; 358extern unsigned long __stop_branch_profile[];
299 359
300static const struct ftrace_pointer ftrace_branch_pos = { 360static int all_branch_stat_headers(struct seq_file *m)
301 .start = __start_branch_profile, 361{
302 .stop = __stop_branch_profile, 362 seq_printf(m, " miss hit %% ");
303 .hit = 1, 363 seq_printf(m, " Function "
304}; 364 " File Line\n"
365 " ------- --------- - "
366 " -------- "
367 " ---- ----\n");
368 return 0;
369}
305 370
306#endif /* CONFIG_PROFILE_ALL_BRANCHES */ 371static void *all_branch_stat_start(void)
372{
373 return __start_branch_profile;
374}
307 375
308extern unsigned long __start_annotated_branch_profile[]; 376static void *
309extern unsigned long __stop_annotated_branch_profile[]; 377all_branch_stat_next(void *v, int idx)
378{
379 struct ftrace_branch_data *p = v;
310 380
311static const struct ftrace_pointer ftrace_annotated_branch_pos = { 381 ++p;
312 .start = __start_annotated_branch_profile,
313 .stop = __stop_annotated_branch_profile,
314};
315 382
316static __init int ftrace_branch_init(void) 383 if ((void *)p >= (void *)__stop_branch_profile)
317{ 384 return NULL;
318 struct dentry *d_tracer;
319 struct dentry *entry;
320 385
321 d_tracer = tracing_init_dentry(); 386 return p;
387}
322 388
323 entry = debugfs_create_file("profile_annotated_branch", 0444, d_tracer, 389static struct tracer_stat all_branch_stats = {
324 (void *)&ftrace_annotated_branch_pos, 390 .name = "branch_all",
325 &tracing_branch_fops); 391 .stat_start = all_branch_stat_start,
326 if (!entry) 392 .stat_next = all_branch_stat_next,
327 pr_warning("Could not create debugfs " 393 .stat_headers = all_branch_stat_headers,
328 "'profile_annotatet_branch' entry\n"); 394 .stat_show = branch_stat_show
395};
329 396
330#ifdef CONFIG_PROFILE_ALL_BRANCHES 397__init static int all_annotated_branch_stats(void)
331 entry = debugfs_create_file("profile_branch", 0444, d_tracer, 398{
332 (void *)&ftrace_branch_pos, 399 int ret;
333 &tracing_branch_fops);
334 if (!entry)
335 pr_warning("Could not create debugfs"
336 " 'profile_branch' entry\n");
337#endif
338 400
401 ret = register_stat_tracer(&all_branch_stats);
402 if (!ret) {
403 printk(KERN_WARNING "Warning: could not register "
404 "all branches stats\n");
405 return 1;
406 }
339 return 0; 407 return 0;
340} 408}
341 409fs_initcall(all_annotated_branch_stats);
342device_initcall(ftrace_branch_init); 410#endif /* CONFIG_PROFILE_ALL_BRANCHES */
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 9236d7e25a16..b3a320f8aba7 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -16,8 +16,17 @@
16 16
17#include "trace.h" 17#include "trace.h"
18 18
19/* function tracing enabled */
20static int ftrace_function_enabled;
21
22static struct trace_array *func_trace;
23
24static void tracing_start_function_trace(void);
25static void tracing_stop_function_trace(void);
26
19static void start_function_trace(struct trace_array *tr) 27static void start_function_trace(struct trace_array *tr)
20{ 28{
29 func_trace = tr;
21 tr->cpu = get_cpu(); 30 tr->cpu = get_cpu();
22 tracing_reset_online_cpus(tr); 31 tracing_reset_online_cpus(tr);
23 put_cpu(); 32 put_cpu();
@@ -48,14 +57,188 @@ static void function_trace_start(struct trace_array *tr)
48 tracing_reset_online_cpus(tr); 57 tracing_reset_online_cpus(tr);
49} 58}
50 59
60static void
61function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
62{
63 struct trace_array *tr = func_trace;
64 struct trace_array_cpu *data;
65 unsigned long flags;
66 long disabled;
67 int cpu, resched;
68 int pc;
69
70 if (unlikely(!ftrace_function_enabled))
71 return;
72
73 pc = preempt_count();
74 resched = ftrace_preempt_disable();
75 local_save_flags(flags);
76 cpu = raw_smp_processor_id();
77 data = tr->data[cpu];
78 disabled = atomic_inc_return(&data->disabled);
79
80 if (likely(disabled == 1))
81 trace_function(tr, data, ip, parent_ip, flags, pc);
82
83 atomic_dec(&data->disabled);
84 ftrace_preempt_enable(resched);
85}
86
87static void
88function_trace_call(unsigned long ip, unsigned long parent_ip)
89{
90 struct trace_array *tr = func_trace;
91 struct trace_array_cpu *data;
92 unsigned long flags;
93 long disabled;
94 int cpu;
95 int pc;
96
97 if (unlikely(!ftrace_function_enabled))
98 return;
99
100 /*
101 * Need to use raw, since this must be called before the
102 * recursive protection is performed.
103 */
104 local_irq_save(flags);
105 cpu = raw_smp_processor_id();
106 data = tr->data[cpu];
107 disabled = atomic_inc_return(&data->disabled);
108
109 if (likely(disabled == 1)) {
110 pc = preempt_count();
111 trace_function(tr, data, ip, parent_ip, flags, pc);
112 }
113
114 atomic_dec(&data->disabled);
115 local_irq_restore(flags);
116}
117
118static void
119function_stack_trace_call(unsigned long ip, unsigned long parent_ip)
120{
121 struct trace_array *tr = func_trace;
122 struct trace_array_cpu *data;
123 unsigned long flags;
124 long disabled;
125 int cpu;
126 int pc;
127
128 if (unlikely(!ftrace_function_enabled))
129 return;
130
131 /*
132 * Need to use raw, since this must be called before the
133 * recursive protection is performed.
134 */
135 local_irq_save(flags);
136 cpu = raw_smp_processor_id();
137 data = tr->data[cpu];
138 disabled = atomic_inc_return(&data->disabled);
139
140 if (likely(disabled == 1)) {
141 pc = preempt_count();
142 trace_function(tr, data, ip, parent_ip, flags, pc);
143 /*
144 * skip over 5 funcs:
145 * __ftrace_trace_stack,
146 * __trace_stack,
147 * function_stack_trace_call
148 * ftrace_list_func
149 * ftrace_call
150 */
151 __trace_stack(tr, data, flags, 5, pc);
152 }
153
154 atomic_dec(&data->disabled);
155 local_irq_restore(flags);
156}
157
158
159static struct ftrace_ops trace_ops __read_mostly =
160{
161 .func = function_trace_call,
162};
163
164static struct ftrace_ops trace_stack_ops __read_mostly =
165{
166 .func = function_stack_trace_call,
167};
168
169/* Our two options */
170enum {
171 TRACE_FUNC_OPT_STACK = 0x1,
172};
173
174static struct tracer_opt func_opts[] = {
175#ifdef CONFIG_STACKTRACE
176 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
177#endif
178 { } /* Always set a last empty entry */
179};
180
181static struct tracer_flags func_flags = {
182 .val = 0, /* By default: all flags disabled */
183 .opts = func_opts
184};
185
186static void tracing_start_function_trace(void)
187{
188 ftrace_function_enabled = 0;
189
190 if (trace_flags & TRACE_ITER_PREEMPTONLY)
191 trace_ops.func = function_trace_call_preempt_only;
192 else
193 trace_ops.func = function_trace_call;
194
195 if (func_flags.val & TRACE_FUNC_OPT_STACK)
196 register_ftrace_function(&trace_stack_ops);
197 else
198 register_ftrace_function(&trace_ops);
199
200 ftrace_function_enabled = 1;
201}
202
203static void tracing_stop_function_trace(void)
204{
205 ftrace_function_enabled = 0;
206 /* OK if they are not registered */
207 unregister_ftrace_function(&trace_stack_ops);
208 unregister_ftrace_function(&trace_ops);
209}
210
211static int func_set_flag(u32 old_flags, u32 bit, int set)
212{
213 if (bit == TRACE_FUNC_OPT_STACK) {
214 /* do nothing if already set */
215 if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
216 return 0;
217
218 if (set) {
219 unregister_ftrace_function(&trace_ops);
220 register_ftrace_function(&trace_stack_ops);
221 } else {
222 unregister_ftrace_function(&trace_stack_ops);
223 register_ftrace_function(&trace_ops);
224 }
225
226 return 0;
227 }
228
229 return -EINVAL;
230}
231
51static struct tracer function_trace __read_mostly = 232static struct tracer function_trace __read_mostly =
52{ 233{
53 .name = "function", 234 .name = "function",
54 .init = function_trace_init, 235 .init = function_trace_init,
55 .reset = function_trace_reset, 236 .reset = function_trace_reset,
56 .start = function_trace_start, 237 .start = function_trace_start,
238 .flags = &func_flags,
239 .set_flag = func_set_flag,
57#ifdef CONFIG_FTRACE_SELFTEST 240#ifdef CONFIG_FTRACE_SELFTEST
58 .selftest = trace_selftest_startup_function, 241 .selftest = trace_selftest_startup_function,
59#endif 242#endif
60}; 243};
61 244
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 930c08e5b38e..3c545984816f 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -12,6 +12,7 @@
12#include <linux/fs.h> 12#include <linux/fs.h>
13 13
14#include "trace.h" 14#include "trace.h"
15#include "trace_output.h"
15 16
16#define TRACE_GRAPH_INDENT 2 17#define TRACE_GRAPH_INDENT 2
17 18
@@ -589,8 +590,11 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
589 if (!ret) 590 if (!ret)
590 return TRACE_TYPE_PARTIAL_LINE; 591 return TRACE_TYPE_PARTIAL_LINE;
591 592
592 if (ent->flags & TRACE_FLAG_CONT) 593 /* Strip ending newline */
593 trace_seq_print_cont(s, iter); 594 if (s->buffer[s->len - 1] == '\n') {
595 s->buffer[s->len - 1] = '\0';
596 s->len--;
597 }
594 598
595 ret = trace_seq_printf(s, " */\n"); 599 ret = trace_seq_printf(s, " */\n");
596 if (!ret) 600 if (!ret)
diff --git a/kernel/trace/trace_hw_branches.c b/kernel/trace/trace_hw_branches.c
index 649df22d435f..df21c1e72b95 100644
--- a/kernel/trace/trace_hw_branches.c
+++ b/kernel/trace/trace_hw_branches.c
@@ -14,6 +14,7 @@
14#include <asm/ds.h> 14#include <asm/ds.h>
15 15
16#include "trace.h" 16#include "trace.h"
17#include "trace_output.h"
17 18
18 19
19#define SIZEOF_BTS (1 << 13) 20#define SIZEOF_BTS (1 << 13)
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index 7c2e326bbc8b..62a78d943534 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -380,6 +380,7 @@ static void stop_irqsoff_tracer(struct trace_array *tr)
380 380
381static void __irqsoff_tracer_init(struct trace_array *tr) 381static void __irqsoff_tracer_init(struct trace_array *tr)
382{ 382{
383 tracing_max_latency = 0;
383 irqsoff_trace = tr; 384 irqsoff_trace = tr;
384 /* make sure that the tracer is visible */ 385 /* make sure that the tracer is visible */
385 smp_wmb(); 386 smp_wmb();
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index fffcb069f1dc..ec78e244242e 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -9,8 +9,10 @@
9#include <linux/kernel.h> 9#include <linux/kernel.h>
10#include <linux/mmiotrace.h> 10#include <linux/mmiotrace.h>
11#include <linux/pci.h> 11#include <linux/pci.h>
12#include <asm/atomic.h>
12 13
13#include "trace.h" 14#include "trace.h"
15#include "trace_output.h"
14 16
15struct header_iter { 17struct header_iter {
16 struct pci_dev *dev; 18 struct pci_dev *dev;
@@ -19,6 +21,7 @@ struct header_iter {
19static struct trace_array *mmio_trace_array; 21static struct trace_array *mmio_trace_array;
20static bool overrun_detected; 22static bool overrun_detected;
21static unsigned long prev_overruns; 23static unsigned long prev_overruns;
24static atomic_t dropped_count;
22 25
23static void mmio_reset_data(struct trace_array *tr) 26static void mmio_reset_data(struct trace_array *tr)
24{ 27{
@@ -121,11 +124,11 @@ static void mmio_close(struct trace_iterator *iter)
121 124
122static unsigned long count_overruns(struct trace_iterator *iter) 125static unsigned long count_overruns(struct trace_iterator *iter)
123{ 126{
124 unsigned long cnt = 0; 127 unsigned long cnt = atomic_xchg(&dropped_count, 0);
125 unsigned long over = ring_buffer_overruns(iter->tr->buffer); 128 unsigned long over = ring_buffer_overruns(iter->tr->buffer);
126 129
127 if (over > prev_overruns) 130 if (over > prev_overruns)
128 cnt = over - prev_overruns; 131 cnt += over - prev_overruns;
129 prev_overruns = over; 132 prev_overruns = over;
130 return cnt; 133 return cnt;
131} 134}
@@ -181,21 +184,22 @@ static enum print_line_t mmio_print_rw(struct trace_iterator *iter)
181 switch (rw->opcode) { 184 switch (rw->opcode) {
182 case MMIO_READ: 185 case MMIO_READ:
183 ret = trace_seq_printf(s, 186 ret = trace_seq_printf(s,
184 "R %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n", 187 "R %d %u.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
185 rw->width, secs, usec_rem, rw->map_id, 188 rw->width, secs, usec_rem, rw->map_id,
186 (unsigned long long)rw->phys, 189 (unsigned long long)rw->phys,
187 rw->value, rw->pc, 0); 190 rw->value, rw->pc, 0);
188 break; 191 break;
189 case MMIO_WRITE: 192 case MMIO_WRITE:
190 ret = trace_seq_printf(s, 193 ret = trace_seq_printf(s,
191 "W %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n", 194 "W %d %u.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
192 rw->width, secs, usec_rem, rw->map_id, 195 rw->width, secs, usec_rem, rw->map_id,
193 (unsigned long long)rw->phys, 196 (unsigned long long)rw->phys,
194 rw->value, rw->pc, 0); 197 rw->value, rw->pc, 0);
195 break; 198 break;
196 case MMIO_UNKNOWN_OP: 199 case MMIO_UNKNOWN_OP:
197 ret = trace_seq_printf(s, 200 ret = trace_seq_printf(s,
198 "UNKNOWN %lu.%06lu %d 0x%llx %02x,%02x,%02x 0x%lx %d\n", 201 "UNKNOWN %u.%06lu %d 0x%llx %02lx,%02lx,"
202 "%02lx 0x%lx %d\n",
199 secs, usec_rem, rw->map_id, 203 secs, usec_rem, rw->map_id,
200 (unsigned long long)rw->phys, 204 (unsigned long long)rw->phys,
201 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff, 205 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff,
@@ -227,14 +231,14 @@ static enum print_line_t mmio_print_map(struct trace_iterator *iter)
227 switch (m->opcode) { 231 switch (m->opcode) {
228 case MMIO_PROBE: 232 case MMIO_PROBE:
229 ret = trace_seq_printf(s, 233 ret = trace_seq_printf(s,
230 "MAP %lu.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n", 234 "MAP %u.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n",
231 secs, usec_rem, m->map_id, 235 secs, usec_rem, m->map_id,
232 (unsigned long long)m->phys, m->virt, m->len, 236 (unsigned long long)m->phys, m->virt, m->len,
233 0UL, 0); 237 0UL, 0);
234 break; 238 break;
235 case MMIO_UNPROBE: 239 case MMIO_UNPROBE:
236 ret = trace_seq_printf(s, 240 ret = trace_seq_printf(s,
237 "UNMAP %lu.%06lu %d 0x%lx %d\n", 241 "UNMAP %u.%06lu %d 0x%lx %d\n",
238 secs, usec_rem, m->map_id, 0UL, 0); 242 secs, usec_rem, m->map_id, 0UL, 0);
239 break; 243 break;
240 default: 244 default:
@@ -258,13 +262,10 @@ static enum print_line_t mmio_print_mark(struct trace_iterator *iter)
258 int ret; 262 int ret;
259 263
260 /* The trailing newline must be in the message. */ 264 /* The trailing newline must be in the message. */
261 ret = trace_seq_printf(s, "MARK %lu.%06lu %s", secs, usec_rem, msg); 265 ret = trace_seq_printf(s, "MARK %u.%06lu %s", secs, usec_rem, msg);
262 if (!ret) 266 if (!ret)
263 return TRACE_TYPE_PARTIAL_LINE; 267 return TRACE_TYPE_PARTIAL_LINE;
264 268
265 if (entry->flags & TRACE_FLAG_CONT)
266 trace_seq_print_cont(s, iter);
267
268 return TRACE_TYPE_HANDLED; 269 return TRACE_TYPE_HANDLED;
269} 270}
270 271
@@ -310,8 +311,10 @@ static void __trace_mmiotrace_rw(struct trace_array *tr,
310 311
311 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 312 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
312 &irq_flags); 313 &irq_flags);
313 if (!event) 314 if (!event) {
315 atomic_inc(&dropped_count);
314 return; 316 return;
317 }
315 entry = ring_buffer_event_data(event); 318 entry = ring_buffer_event_data(event);
316 tracing_generic_entry_update(&entry->ent, 0, preempt_count()); 319 tracing_generic_entry_update(&entry->ent, 0, preempt_count());
317 entry->ent.type = TRACE_MMIO_RW; 320 entry->ent.type = TRACE_MMIO_RW;
@@ -338,8 +341,10 @@ static void __trace_mmiotrace_map(struct trace_array *tr,
338 341
339 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 342 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
340 &irq_flags); 343 &irq_flags);
341 if (!event) 344 if (!event) {
345 atomic_inc(&dropped_count);
342 return; 346 return;
347 }
343 entry = ring_buffer_event_data(event); 348 entry = ring_buffer_event_data(event);
344 tracing_generic_entry_update(&entry->ent, 0, preempt_count()); 349 tracing_generic_entry_update(&entry->ent, 0, preempt_count());
345 entry->ent.type = TRACE_MMIO_MAP; 350 entry->ent.type = TRACE_MMIO_MAP;
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
new file mode 100644
index 000000000000..1a4e144a9f8f
--- /dev/null
+++ b/kernel/trace/trace_output.c
@@ -0,0 +1,829 @@
1/*
2 * trace_output.c
3 *
4 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
5 *
6 */
7
8#include <linux/module.h>
9#include <linux/mutex.h>
10#include <linux/ftrace.h>
11
12#include "trace_output.h"
13
14/* must be a power of 2 */
15#define EVENT_HASHSIZE 128
16
17static DEFINE_MUTEX(trace_event_mutex);
18static struct hlist_head event_hash[EVENT_HASHSIZE] __read_mostly;
19
20static int next_event_type = __TRACE_LAST_TYPE + 1;
21
22/**
23 * trace_seq_printf - sequence printing of trace information
24 * @s: trace sequence descriptor
25 * @fmt: printf format string
26 *
27 * The tracer may use either sequence operations or its own
28 * copy to user routines. To simplify formating of a trace
29 * trace_seq_printf is used to store strings into a special
30 * buffer (@s). Then the output may be either used by
31 * the sequencer or pulled into another buffer.
32 */
33int
34trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
35{
36 int len = (PAGE_SIZE - 1) - s->len;
37 va_list ap;
38 int ret;
39
40 if (!len)
41 return 0;
42
43 va_start(ap, fmt);
44 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
45 va_end(ap);
46
47 /* If we can't write it all, don't bother writing anything */
48 if (ret >= len)
49 return 0;
50
51 s->len += ret;
52
53 return len;
54}
55
56/**
57 * trace_seq_puts - trace sequence printing of simple string
58 * @s: trace sequence descriptor
59 * @str: simple string to record
60 *
61 * The tracer may use either the sequence operations or its own
62 * copy to user routines. This function records a simple string
63 * into a special buffer (@s) for later retrieval by a sequencer
64 * or other mechanism.
65 */
66int trace_seq_puts(struct trace_seq *s, const char *str)
67{
68 int len = strlen(str);
69
70 if (len > ((PAGE_SIZE - 1) - s->len))
71 return 0;
72
73 memcpy(s->buffer + s->len, str, len);
74 s->len += len;
75
76 return len;
77}
78
79int trace_seq_putc(struct trace_seq *s, unsigned char c)
80{
81 if (s->len >= (PAGE_SIZE - 1))
82 return 0;
83
84 s->buffer[s->len++] = c;
85
86 return 1;
87}
88
89int trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
90{
91 if (len > ((PAGE_SIZE - 1) - s->len))
92 return 0;
93
94 memcpy(s->buffer + s->len, mem, len);
95 s->len += len;
96
97 return len;
98}
99
100int trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
101{
102 unsigned char hex[HEX_CHARS];
103 unsigned char *data = mem;
104 int i, j;
105
106#ifdef __BIG_ENDIAN
107 for (i = 0, j = 0; i < len; i++) {
108#else
109 for (i = len-1, j = 0; i >= 0; i--) {
110#endif
111 hex[j++] = hex_asc_hi(data[i]);
112 hex[j++] = hex_asc_lo(data[i]);
113 }
114 hex[j++] = ' ';
115
116 return trace_seq_putmem(s, hex, j);
117}
118
119int trace_seq_path(struct trace_seq *s, struct path *path)
120{
121 unsigned char *p;
122
123 if (s->len >= (PAGE_SIZE - 1))
124 return 0;
125 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len);
126 if (!IS_ERR(p)) {
127 p = mangle_path(s->buffer + s->len, p, "\n");
128 if (p) {
129 s->len = p - s->buffer;
130 return 1;
131 }
132 } else {
133 s->buffer[s->len++] = '?';
134 return 1;
135 }
136
137 return 0;
138}
139
140#ifdef CONFIG_KRETPROBES
141static inline const char *kretprobed(const char *name)
142{
143 static const char tramp_name[] = "kretprobe_trampoline";
144 int size = sizeof(tramp_name);
145
146 if (strncmp(tramp_name, name, size) == 0)
147 return "[unknown/kretprobe'd]";
148 return name;
149}
150#else
151static inline const char *kretprobed(const char *name)
152{
153 return name;
154}
155#endif /* CONFIG_KRETPROBES */
156
157static int
158seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
159{
160#ifdef CONFIG_KALLSYMS
161 char str[KSYM_SYMBOL_LEN];
162 const char *name;
163
164 kallsyms_lookup(address, NULL, NULL, NULL, str);
165
166 name = kretprobed(str);
167
168 return trace_seq_printf(s, fmt, name);
169#endif
170 return 1;
171}
172
173static int
174seq_print_sym_offset(struct trace_seq *s, const char *fmt,
175 unsigned long address)
176{
177#ifdef CONFIG_KALLSYMS
178 char str[KSYM_SYMBOL_LEN];
179 const char *name;
180
181 sprint_symbol(str, address);
182 name = kretprobed(str);
183
184 return trace_seq_printf(s, fmt, name);
185#endif
186 return 1;
187}
188
189#ifndef CONFIG_64BIT
190# define IP_FMT "%08lx"
191#else
192# define IP_FMT "%016lx"
193#endif
194
195int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
196 unsigned long ip, unsigned long sym_flags)
197{
198 struct file *file = NULL;
199 unsigned long vmstart = 0;
200 int ret = 1;
201
202 if (mm) {
203 const struct vm_area_struct *vma;
204
205 down_read(&mm->mmap_sem);
206 vma = find_vma(mm, ip);
207 if (vma) {
208 file = vma->vm_file;
209 vmstart = vma->vm_start;
210 }
211 if (file) {
212 ret = trace_seq_path(s, &file->f_path);
213 if (ret)
214 ret = trace_seq_printf(s, "[+0x%lx]",
215 ip - vmstart);
216 }
217 up_read(&mm->mmap_sem);
218 }
219 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
220 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
221 return ret;
222}
223
224int
225seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
226 unsigned long sym_flags)
227{
228 struct mm_struct *mm = NULL;
229 int ret = 1;
230 unsigned int i;
231
232 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
233 struct task_struct *task;
234 /*
235 * we do the lookup on the thread group leader,
236 * since individual threads might have already quit!
237 */
238 rcu_read_lock();
239 task = find_task_by_vpid(entry->ent.tgid);
240 if (task)
241 mm = get_task_mm(task);
242 rcu_read_unlock();
243 }
244
245 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
246 unsigned long ip = entry->caller[i];
247
248 if (ip == ULONG_MAX || !ret)
249 break;
250 if (i && ret)
251 ret = trace_seq_puts(s, " <- ");
252 if (!ip) {
253 if (ret)
254 ret = trace_seq_puts(s, "??");
255 continue;
256 }
257 if (!ret)
258 break;
259 if (ret)
260 ret = seq_print_user_ip(s, mm, ip, sym_flags);
261 }
262
263 if (mm)
264 mmput(mm);
265 return ret;
266}
267
268int
269seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
270{
271 int ret;
272
273 if (!ip)
274 return trace_seq_printf(s, "0");
275
276 if (sym_flags & TRACE_ITER_SYM_OFFSET)
277 ret = seq_print_sym_offset(s, "%s", ip);
278 else
279 ret = seq_print_sym_short(s, "%s", ip);
280
281 if (!ret)
282 return 0;
283
284 if (sym_flags & TRACE_ITER_SYM_ADDR)
285 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
286 return ret;
287}
288
289static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
290
291static int task_state_char(unsigned long state)
292{
293 int bit = state ? __ffs(state) + 1 : 0;
294
295 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
296}
297
298/**
299 * ftrace_find_event - find a registered event
300 * @type: the type of event to look for
301 *
302 * Returns an event of type @type otherwise NULL
303 */
304struct trace_event *ftrace_find_event(int type)
305{
306 struct trace_event *event;
307 struct hlist_node *n;
308 unsigned key;
309
310 key = type & (EVENT_HASHSIZE - 1);
311
312 hlist_for_each_entry_rcu(event, n, &event_hash[key], node) {
313 if (event->type == type)
314 return event;
315 }
316
317 return NULL;
318}
319
320/**
321 * register_ftrace_event - register output for an event type
322 * @event: the event type to register
323 *
324 * Event types are stored in a hash and this hash is used to
325 * find a way to print an event. If the @event->type is set
326 * then it will use that type, otherwise it will assign a
327 * type to use.
328 *
329 * If you assign your own type, please make sure it is added
330 * to the trace_type enum in trace.h, to avoid collisions
331 * with the dynamic types.
332 *
333 * Returns the event type number or zero on error.
334 */
335int register_ftrace_event(struct trace_event *event)
336{
337 unsigned key;
338 int ret = 0;
339
340 mutex_lock(&trace_event_mutex);
341
342 if (!event->type)
343 event->type = next_event_type++;
344 else if (event->type > __TRACE_LAST_TYPE) {
345 printk(KERN_WARNING "Need to add type to trace.h\n");
346 WARN_ON(1);
347 }
348
349 if (ftrace_find_event(event->type))
350 goto out;
351
352 key = event->type & (EVENT_HASHSIZE - 1);
353
354 hlist_add_head_rcu(&event->node, &event_hash[key]);
355
356 ret = event->type;
357 out:
358 mutex_unlock(&trace_event_mutex);
359
360 return ret;
361}
362
363/**
364 * unregister_ftrace_event - remove a no longer used event
365 * @event: the event to remove
366 */
367int unregister_ftrace_event(struct trace_event *event)
368{
369 mutex_lock(&trace_event_mutex);
370 hlist_del(&event->node);
371 mutex_unlock(&trace_event_mutex);
372
373 return 0;
374}
375
376/*
377 * Standard events
378 */
379
380int
381trace_nop_print(struct trace_seq *s, struct trace_entry *entry, int flags)
382{
383 return 0;
384}
385
386/* TRACE_FN */
387static int
388trace_fn_latency(struct trace_seq *s, struct trace_entry *entry, int flags)
389{
390 struct ftrace_entry *field;
391
392 trace_assign_type(field, entry);
393
394 if (!seq_print_ip_sym(s, field->ip, flags))
395 goto partial;
396 if (!trace_seq_puts(s, " ("))
397 goto partial;
398 if (!seq_print_ip_sym(s, field->parent_ip, flags))
399 goto partial;
400 if (!trace_seq_puts(s, ")\n"))
401 goto partial;
402
403 return 0;
404
405 partial:
406 return TRACE_TYPE_PARTIAL_LINE;
407}
408
409static int
410trace_fn_trace(struct trace_seq *s, struct trace_entry *entry, int flags)
411{
412 struct ftrace_entry *field;
413
414 trace_assign_type(field, entry);
415
416 if (!seq_print_ip_sym(s, field->ip, flags))
417 goto partial;
418
419 if ((flags & TRACE_ITER_PRINT_PARENT) && field->parent_ip) {
420 if (!trace_seq_printf(s, " <-"))
421 goto partial;
422 if (!seq_print_ip_sym(s,
423 field->parent_ip,
424 flags))
425 goto partial;
426 }
427 if (!trace_seq_printf(s, "\n"))
428 goto partial;
429
430 return 0;
431
432 partial:
433 return TRACE_TYPE_PARTIAL_LINE;
434}
435
436static int
437trace_fn_raw(struct trace_seq *s, struct trace_entry *entry, int flags)
438{
439 struct ftrace_entry *field;
440
441 trace_assign_type(field, entry);
442
443 if (!trace_seq_printf(s, "%lx %lx\n",
444 field->ip,
445 field->parent_ip))
446 return TRACE_TYPE_PARTIAL_LINE;
447
448 return 0;
449}
450
451static int
452trace_fn_hex(struct trace_seq *s, struct trace_entry *entry, int flags)
453{
454 struct ftrace_entry *field;
455
456 trace_assign_type(field, entry);
457
458 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
459 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
460
461 return 0;
462}
463
464static int
465trace_fn_bin(struct trace_seq *s, struct trace_entry *entry, int flags)
466{
467 struct ftrace_entry *field;
468
469 trace_assign_type(field, entry);
470
471 SEQ_PUT_FIELD_RET(s, field->ip);
472 SEQ_PUT_FIELD_RET(s, field->parent_ip);
473
474 return 0;
475}
476
477static struct trace_event trace_fn_event = {
478 .type = TRACE_FN,
479 .trace = trace_fn_trace,
480 .latency_trace = trace_fn_latency,
481 .raw = trace_fn_raw,
482 .hex = trace_fn_hex,
483 .binary = trace_fn_bin,
484};
485
486/* TRACE_CTX an TRACE_WAKE */
487static int
488trace_ctxwake_print(struct trace_seq *s, struct trace_entry *entry, int flags,
489 char *delim)
490{
491 struct ctx_switch_entry *field;
492 char *comm;
493 int S, T;
494
495 trace_assign_type(field, entry);
496
497 T = task_state_char(field->next_state);
498 S = task_state_char(field->prev_state);
499 comm = trace_find_cmdline(field->next_pid);
500 if (!trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
501 field->prev_pid,
502 field->prev_prio,
503 S, delim,
504 field->next_cpu,
505 field->next_pid,
506 field->next_prio,
507 T, comm))
508 return TRACE_TYPE_PARTIAL_LINE;
509
510 return 0;
511}
512
513static int
514trace_ctx_print(struct trace_seq *s, struct trace_entry *entry, int flags)
515{
516 return trace_ctxwake_print(s, entry, flags, "==>");
517}
518
519static int
520trace_wake_print(struct trace_seq *s, struct trace_entry *entry, int flags)
521{
522 return trace_ctxwake_print(s, entry, flags, " +");
523}
524
525static int
526trace_ctxwake_raw(struct trace_seq *s, struct trace_entry *entry, int flags,
527 char S)
528{
529 struct ctx_switch_entry *field;
530 int T;
531
532 trace_assign_type(field, entry);
533
534 if (!S)
535 task_state_char(field->prev_state);
536 T = task_state_char(field->next_state);
537 if (!trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
538 field->prev_pid,
539 field->prev_prio,
540 S,
541 field->next_cpu,
542 field->next_pid,
543 field->next_prio,
544 T))
545 return TRACE_TYPE_PARTIAL_LINE;
546
547 return 0;
548}
549
550static int
551trace_ctx_raw(struct trace_seq *s, struct trace_entry *entry, int flags)
552{
553 return trace_ctxwake_raw(s, entry, flags, 0);
554}
555
556static int
557trace_wake_raw(struct trace_seq *s, struct trace_entry *entry, int flags)
558{
559 return trace_ctxwake_raw(s, entry, flags, '+');
560}
561
562
563static int
564trace_ctxwake_hex(struct trace_seq *s, struct trace_entry *entry, int flags,
565 char S)
566{
567 struct ctx_switch_entry *field;
568 int T;
569
570 trace_assign_type(field, entry);
571
572 if (!S)
573 task_state_char(field->prev_state);
574 T = task_state_char(field->next_state);
575
576 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
577 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
578 SEQ_PUT_HEX_FIELD_RET(s, S);
579 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
580 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
581 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
582 SEQ_PUT_HEX_FIELD_RET(s, T);
583
584 return 0;
585}
586
587static int
588trace_ctx_hex(struct trace_seq *s, struct trace_entry *entry, int flags)
589{
590 return trace_ctxwake_hex(s, entry, flags, 0);
591}
592
593static int
594trace_wake_hex(struct trace_seq *s, struct trace_entry *entry, int flags)
595{
596 return trace_ctxwake_hex(s, entry, flags, '+');
597}
598
599static int
600trace_ctxwake_bin(struct trace_seq *s, struct trace_entry *entry, int flags)
601{
602 struct ctx_switch_entry *field;
603
604 trace_assign_type(field, entry);
605
606 SEQ_PUT_FIELD_RET(s, field->prev_pid);
607 SEQ_PUT_FIELD_RET(s, field->prev_prio);
608 SEQ_PUT_FIELD_RET(s, field->prev_state);
609 SEQ_PUT_FIELD_RET(s, field->next_pid);
610 SEQ_PUT_FIELD_RET(s, field->next_prio);
611 SEQ_PUT_FIELD_RET(s, field->next_state);
612
613 return 0;
614}
615
616static struct trace_event trace_ctx_event = {
617 .type = TRACE_CTX,
618 .trace = trace_ctx_print,
619 .latency_trace = trace_ctx_print,
620 .raw = trace_ctx_raw,
621 .hex = trace_ctx_hex,
622 .binary = trace_ctxwake_bin,
623};
624
625static struct trace_event trace_wake_event = {
626 .type = TRACE_WAKE,
627 .trace = trace_wake_print,
628 .latency_trace = trace_wake_print,
629 .raw = trace_wake_raw,
630 .hex = trace_wake_hex,
631 .binary = trace_ctxwake_bin,
632};
633
634/* TRACE_SPECIAL */
635static int
636trace_special_print(struct trace_seq *s, struct trace_entry *entry, int flags)
637{
638 struct special_entry *field;
639
640 trace_assign_type(field, entry);
641
642 if (!trace_seq_printf(s, "# %ld %ld %ld\n",
643 field->arg1,
644 field->arg2,
645 field->arg3))
646 return TRACE_TYPE_PARTIAL_LINE;
647
648 return 0;
649}
650
651static int
652trace_special_hex(struct trace_seq *s, struct trace_entry *entry, int flags)
653{
654 struct special_entry *field;
655
656 trace_assign_type(field, entry);
657
658 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
659 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
660 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
661
662 return 0;
663}
664
665static int
666trace_special_bin(struct trace_seq *s, struct trace_entry *entry, int flags)
667{
668 struct special_entry *field;
669
670 trace_assign_type(field, entry);
671
672 SEQ_PUT_FIELD_RET(s, field->arg1);
673 SEQ_PUT_FIELD_RET(s, field->arg2);
674 SEQ_PUT_FIELD_RET(s, field->arg3);
675
676 return 0;
677}
678
679static struct trace_event trace_special_event = {
680 .type = TRACE_SPECIAL,
681 .trace = trace_special_print,
682 .latency_trace = trace_special_print,
683 .raw = trace_special_print,
684 .hex = trace_special_hex,
685 .binary = trace_special_bin,
686};
687
688/* TRACE_STACK */
689
690static int
691trace_stack_print(struct trace_seq *s, struct trace_entry *entry, int flags)
692{
693 struct stack_entry *field;
694 int i;
695
696 trace_assign_type(field, entry);
697
698 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
699 if (i) {
700 if (!trace_seq_puts(s, " <= "))
701 goto partial;
702
703 if (!seq_print_ip_sym(s, field->caller[i], flags))
704 goto partial;
705 }
706 if (!trace_seq_puts(s, "\n"))
707 goto partial;
708 }
709
710 return 0;
711
712 partial:
713 return TRACE_TYPE_PARTIAL_LINE;
714}
715
716static struct trace_event trace_stack_event = {
717 .type = TRACE_STACK,
718 .trace = trace_stack_print,
719 .latency_trace = trace_stack_print,
720 .raw = trace_special_print,
721 .hex = trace_special_hex,
722 .binary = trace_special_bin,
723};
724
725/* TRACE_USER_STACK */
726static int
727trace_user_stack_print(struct trace_seq *s, struct trace_entry *entry,
728 int flags)
729{
730 struct userstack_entry *field;
731
732 trace_assign_type(field, entry);
733
734 if (!seq_print_userip_objs(field, s, flags))
735 goto partial;
736
737 if (!trace_seq_putc(s, '\n'))
738 goto partial;
739
740 return 0;
741
742 partial:
743 return TRACE_TYPE_PARTIAL_LINE;
744}
745
746static struct trace_event trace_user_stack_event = {
747 .type = TRACE_USER_STACK,
748 .trace = trace_user_stack_print,
749 .latency_trace = trace_user_stack_print,
750 .raw = trace_special_print,
751 .hex = trace_special_hex,
752 .binary = trace_special_bin,
753};
754
755/* TRACE_PRINT */
756static int
757trace_print_print(struct trace_seq *s, struct trace_entry *entry, int flags)
758{
759 struct print_entry *field;
760
761 trace_assign_type(field, entry);
762
763 if (!seq_print_ip_sym(s, field->ip, flags))
764 goto partial;
765
766 if (!trace_seq_printf(s, ": %s", field->buf))
767 goto partial;
768
769 return 0;
770
771 partial:
772 return TRACE_TYPE_PARTIAL_LINE;
773}
774
775static int
776trace_print_raw(struct trace_seq *s, struct trace_entry *entry, int flags)
777{
778 struct print_entry *field;
779
780 trace_assign_type(field, entry);
781
782 if (!trace_seq_printf(s, "# %lx %s", field->ip, field->buf))
783 goto partial;
784
785 return 0;
786
787 partial:
788 return TRACE_TYPE_PARTIAL_LINE;
789}
790
791static struct trace_event trace_print_event = {
792 .type = TRACE_PRINT,
793 .trace = trace_print_print,
794 .latency_trace = trace_print_print,
795 .raw = trace_print_raw,
796 .hex = trace_nop_print,
797 .binary = trace_nop_print,
798};
799
800static struct trace_event *events[] __initdata = {
801 &trace_fn_event,
802 &trace_ctx_event,
803 &trace_wake_event,
804 &trace_special_event,
805 &trace_stack_event,
806 &trace_user_stack_event,
807 &trace_print_event,
808 NULL
809};
810
811__init static int init_events(void)
812{
813 struct trace_event *event;
814 int i, ret;
815
816 for (i = 0; events[i]; i++) {
817 event = events[i];
818
819 ret = register_ftrace_event(event);
820 if (!ret) {
821 printk(KERN_WARNING "event %d failed to register\n",
822 event->type);
823 WARN_ON_ONCE(1);
824 }
825 }
826
827 return 0;
828}
829device_initcall(init_events);
diff --git a/kernel/trace/trace_output.h b/kernel/trace/trace_output.h
new file mode 100644
index 000000000000..1cbab5e3dc99
--- /dev/null
+++ b/kernel/trace/trace_output.h
@@ -0,0 +1,60 @@
1#ifndef __TRACE_EVENTS_H
2#define __TRACE_EVENTS_H
3
4#include "trace.h"
5
6typedef int (*trace_print_func)(struct trace_seq *s, struct trace_entry *entry,
7 int flags);
8
9struct trace_event {
10 struct hlist_node node;
11 int type;
12 trace_print_func trace;
13 trace_print_func latency_trace;
14 trace_print_func raw;
15 trace_print_func hex;
16 trace_print_func binary;
17};
18
19extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
20 __attribute__ ((format (printf, 2, 3)));
21extern int
22seq_print_ip_sym(struct trace_seq *s, unsigned long ip,
23 unsigned long sym_flags);
24extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
25 size_t cnt);
26int trace_seq_puts(struct trace_seq *s, const char *str);
27int trace_seq_putc(struct trace_seq *s, unsigned char c);
28int trace_seq_putmem(struct trace_seq *s, void *mem, size_t len);
29int trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len);
30int trace_seq_path(struct trace_seq *s, struct path *path);
31int seq_print_userip_objs(const struct userstack_entry *entry,
32 struct trace_seq *s, unsigned long sym_flags);
33int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
34 unsigned long ip, unsigned long sym_flags);
35
36struct trace_event *ftrace_find_event(int type);
37int register_ftrace_event(struct trace_event *event);
38int unregister_ftrace_event(struct trace_event *event);
39
40int
41trace_nop_print(struct trace_seq *s, struct trace_entry *entry, int flags);
42
43#define MAX_MEMHEX_BYTES 8
44#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
45
46#define SEQ_PUT_FIELD_RET(s, x) \
47do { \
48 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
49 return TRACE_TYPE_PARTIAL_LINE; \
50} while (0)
51
52#define SEQ_PUT_HEX_FIELD_RET(s, x) \
53do { \
54 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
55 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
56 return TRACE_TYPE_PARTIAL_LINE; \
57} while (0)
58
59#endif
60
diff --git a/kernel/trace/trace_power.c b/kernel/trace/trace_power.c
index 7bda248daf55..faa6ab7a1f5c 100644
--- a/kernel/trace/trace_power.c
+++ b/kernel/trace/trace_power.c
@@ -16,6 +16,7 @@
16#include <linux/module.h> 16#include <linux/module.h>
17 17
18#include "trace.h" 18#include "trace.h"
19#include "trace_output.h"
19 20
20static struct trace_array *power_trace; 21static struct trace_array *power_trace;
21static int __read_mostly trace_power_enabled; 22static int __read_mostly trace_power_enabled;
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 43586b689e31..42ae1e77b6b3 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -333,6 +333,7 @@ static void stop_wakeup_tracer(struct trace_array *tr)
333 333
334static int wakeup_tracer_init(struct trace_array *tr) 334static int wakeup_tracer_init(struct trace_array *tr)
335{ 335{
336 tracing_max_latency = 0;
336 wakeup_trace = tr; 337 wakeup_trace = tr;
337 start_wakeup_tracer(tr); 338 start_wakeup_tracer(tr);
338 return 0; 339 return 0;
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 88c8eb70f54a..5013812578b1 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -9,7 +9,6 @@ static inline int trace_valid_entry(struct trace_entry *entry)
9 case TRACE_FN: 9 case TRACE_FN:
10 case TRACE_CTX: 10 case TRACE_CTX:
11 case TRACE_WAKE: 11 case TRACE_WAKE:
12 case TRACE_CONT:
13 case TRACE_STACK: 12 case TRACE_STACK:
14 case TRACE_PRINT: 13 case TRACE_PRINT:
15 case TRACE_SPECIAL: 14 case TRACE_SPECIAL:
diff --git a/kernel/trace/trace_stat.c b/kernel/trace/trace_stat.c
new file mode 100644
index 000000000000..eae9cef39291
--- /dev/null
+++ b/kernel/trace/trace_stat.c
@@ -0,0 +1,319 @@
1/*
2 * Infrastructure for statistic tracing (histogram output).
3 *
4 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
5 *
6 * Based on the code from trace_branch.c which is
7 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
8 *
9 */
10
11
12#include <linux/list.h>
13#include <linux/debugfs.h>
14#include "trace_stat.h"
15#include "trace.h"
16
17
18/* List of stat entries from a tracer */
19struct trace_stat_list {
20 struct list_head list;
21 void *stat;
22};
23
24/* A stat session is the stats output in one file */
25struct tracer_stat_session {
26 struct list_head session_list;
27 struct tracer_stat *ts;
28 struct list_head stat_list;
29 struct mutex stat_mutex;
30 struct dentry *file;
31};
32
33/* All of the sessions currently in use. Each stat file embeed one session */
34static LIST_HEAD(all_stat_sessions);
35static DEFINE_MUTEX(all_stat_sessions_mutex);
36
37/* The root directory for all stat files */
38static struct dentry *stat_dir;
39
40
41static void reset_stat_session(struct tracer_stat_session *session)
42{
43 struct trace_stat_list *node, *next;
44
45 list_for_each_entry_safe(node, next, &session->stat_list, list)
46 kfree(node);
47
48 INIT_LIST_HEAD(&session->stat_list);
49}
50
51static void destroy_session(struct tracer_stat_session *session)
52{
53 debugfs_remove(session->file);
54 reset_stat_session(session);
55 mutex_destroy(&session->stat_mutex);
56 kfree(session);
57}
58
59/*
60 * For tracers that don't provide a stat_cmp callback.
61 * This one will force an immediate insertion on tail of
62 * the list.
63 */
64static int dummy_cmp(void *p1, void *p2)
65{
66 return 1;
67}
68
69/*
70 * Initialize the stat list at each trace_stat file opening.
71 * All of these copies and sorting are required on all opening
72 * since the stats could have changed between two file sessions.
73 */
74static int stat_seq_init(struct tracer_stat_session *session)
75{
76 struct trace_stat_list *iter_entry, *new_entry;
77 struct tracer_stat *ts = session->ts;
78 void *prev_stat;
79 int ret = 0;
80 int i;
81
82 mutex_lock(&session->stat_mutex);
83 reset_stat_session(session);
84
85 if (!ts->stat_cmp)
86 ts->stat_cmp = dummy_cmp;
87
88 /*
89 * The first entry. Actually this is the second, but the first
90 * one (the stat_list head) is pointless.
91 */
92 new_entry = kmalloc(sizeof(struct trace_stat_list), GFP_KERNEL);
93 if (!new_entry) {
94 ret = -ENOMEM;
95 goto exit;
96 }
97
98 INIT_LIST_HEAD(&new_entry->list);
99
100 list_add(&new_entry->list, &session->stat_list);
101
102 new_entry->stat = ts->stat_start();
103 prev_stat = new_entry->stat;
104
105 /*
106 * Iterate over the tracer stat entries and store them in a sorted
107 * list.
108 */
109 for (i = 1; ; i++) {
110 new_entry = kmalloc(sizeof(struct trace_stat_list), GFP_KERNEL);
111 if (!new_entry) {
112 ret = -ENOMEM;
113 goto exit_free_list;
114 }
115
116 INIT_LIST_HEAD(&new_entry->list);
117 new_entry->stat = ts->stat_next(prev_stat, i);
118
119 /* End of insertion */
120 if (!new_entry->stat)
121 break;
122
123 list_for_each_entry(iter_entry, &session->stat_list, list) {
124
125 /* Insertion with a descendent sorting */
126 if (ts->stat_cmp(new_entry->stat,
127 iter_entry->stat) > 0) {
128
129 list_add_tail(&new_entry->list,
130 &iter_entry->list);
131 break;
132
133 /* The current smaller value */
134 } else if (list_is_last(&iter_entry->list,
135 &session->stat_list)) {
136 list_add(&new_entry->list, &iter_entry->list);
137 break;
138 }
139 }
140
141 prev_stat = new_entry->stat;
142 }
143exit:
144 mutex_unlock(&session->stat_mutex);
145 return ret;
146
147exit_free_list:
148 reset_stat_session(session);
149 mutex_unlock(&session->stat_mutex);
150 return ret;
151}
152
153
154static void *stat_seq_start(struct seq_file *s, loff_t *pos)
155{
156 struct tracer_stat_session *session = s->private;
157
158 /* Prevent from tracer switch or stat_list modification */
159 mutex_lock(&session->stat_mutex);
160
161 /* If we are in the beginning of the file, print the headers */
162 if (!*pos && session->ts->stat_headers)
163 session->ts->stat_headers(s);
164
165 return seq_list_start(&session->stat_list, *pos);
166}
167
168static void *stat_seq_next(struct seq_file *s, void *p, loff_t *pos)
169{
170 struct tracer_stat_session *session = s->private;
171
172 return seq_list_next(p, &session->stat_list, pos);
173}
174
175static void stat_seq_stop(struct seq_file *s, void *p)
176{
177 struct tracer_stat_session *session = s->private;
178 mutex_unlock(&session->stat_mutex);
179}
180
181static int stat_seq_show(struct seq_file *s, void *v)
182{
183 struct tracer_stat_session *session = s->private;
184 struct trace_stat_list *l = list_entry(v, struct trace_stat_list, list);
185
186 return session->ts->stat_show(s, l->stat);
187}
188
189static const struct seq_operations trace_stat_seq_ops = {
190 .start = stat_seq_start,
191 .next = stat_seq_next,
192 .stop = stat_seq_stop,
193 .show = stat_seq_show
194};
195
196/* The session stat is refilled and resorted at each stat file opening */
197static int tracing_stat_open(struct inode *inode, struct file *file)
198{
199 int ret;
200
201 struct tracer_stat_session *session = inode->i_private;
202
203 ret = seq_open(file, &trace_stat_seq_ops);
204 if (!ret) {
205 struct seq_file *m = file->private_data;
206 m->private = session;
207 ret = stat_seq_init(session);
208 }
209
210 return ret;
211}
212
213/*
214 * Avoid consuming memory with our now useless list.
215 */
216static int tracing_stat_release(struct inode *i, struct file *f)
217{
218 struct tracer_stat_session *session = i->i_private;
219
220 mutex_lock(&session->stat_mutex);
221 reset_stat_session(session);
222 mutex_unlock(&session->stat_mutex);
223
224 return 0;
225}
226
227static const struct file_operations tracing_stat_fops = {
228 .open = tracing_stat_open,
229 .read = seq_read,
230 .llseek = seq_lseek,
231 .release = tracing_stat_release
232};
233
234static int tracing_stat_init(void)
235{
236 struct dentry *d_tracing;
237
238 d_tracing = tracing_init_dentry();
239
240 stat_dir = debugfs_create_dir("trace_stat", d_tracing);
241 if (!stat_dir)
242 pr_warning("Could not create debugfs "
243 "'trace_stat' entry\n");
244 return 0;
245}
246
247static int init_stat_file(struct tracer_stat_session *session)
248{
249 if (!stat_dir && tracing_stat_init())
250 return -ENODEV;
251
252 session->file = debugfs_create_file(session->ts->name, 0644,
253 stat_dir,
254 session, &tracing_stat_fops);
255 if (!session->file)
256 return -ENOMEM;
257 return 0;
258}
259
260int register_stat_tracer(struct tracer_stat *trace)
261{
262 struct tracer_stat_session *session, *node, *tmp;
263 int ret;
264
265 if (!trace)
266 return -EINVAL;
267
268 if (!trace->stat_start || !trace->stat_next || !trace->stat_show)
269 return -EINVAL;
270
271 /* Already registered? */
272 mutex_lock(&all_stat_sessions_mutex);
273 list_for_each_entry_safe(node, tmp, &all_stat_sessions, session_list) {
274 if (node->ts == trace) {
275 mutex_unlock(&all_stat_sessions_mutex);
276 return -EINVAL;
277 }
278 }
279 mutex_unlock(&all_stat_sessions_mutex);
280
281 /* Init the session */
282 session = kmalloc(sizeof(struct tracer_stat_session), GFP_KERNEL);
283 if (!session)
284 return -ENOMEM;
285
286 session->ts = trace;
287 INIT_LIST_HEAD(&session->session_list);
288 INIT_LIST_HEAD(&session->stat_list);
289 mutex_init(&session->stat_mutex);
290 session->file = NULL;
291
292 ret = init_stat_file(session);
293 if (ret) {
294 destroy_session(session);
295 return ret;
296 }
297
298 /* Register */
299 mutex_lock(&all_stat_sessions_mutex);
300 list_add_tail(&session->session_list, &all_stat_sessions);
301 mutex_unlock(&all_stat_sessions_mutex);
302
303 return 0;
304}
305
306void unregister_stat_tracer(struct tracer_stat *trace)
307{
308 struct tracer_stat_session *node, *tmp;
309
310 mutex_lock(&all_stat_sessions_mutex);
311 list_for_each_entry_safe(node, tmp, &all_stat_sessions, session_list) {
312 if (node->ts == trace) {
313 list_del(&node->session_list);
314 destroy_session(node);
315 break;
316 }
317 }
318 mutex_unlock(&all_stat_sessions_mutex);
319}
diff --git a/kernel/trace/trace_stat.h b/kernel/trace/trace_stat.h
new file mode 100644
index 000000000000..202274cf7f3d
--- /dev/null
+++ b/kernel/trace/trace_stat.h
@@ -0,0 +1,31 @@
1#ifndef __TRACE_STAT_H
2#define __TRACE_STAT_H
3
4#include <linux/seq_file.h>
5
6/*
7 * If you want to provide a stat file (one-shot statistics), fill
8 * an iterator with stat_start/stat_next and a stat_show callbacks.
9 * The others callbacks are optional.
10 */
11struct tracer_stat {
12 /* The name of your stat file */
13 const char *name;
14 /* Iteration over statistic entries */
15 void *(*stat_start)(void);
16 void *(*stat_next)(void *prev, int idx);
17 /* Compare two entries for stats sorting */
18 int (*stat_cmp)(void *p1, void *p2);
19 /* Print a stat entry */
20 int (*stat_show)(struct seq_file *s, void *p);
21 /* Print the headers of your stat entries */
22 int (*stat_headers)(struct seq_file *s);
23};
24
25/*
26 * Destroy or create a stat file
27 */
28extern int register_stat_tracer(struct tracer_stat *trace);
29extern void unregister_stat_tracer(struct tracer_stat *trace);
30
31#endif /* __TRACE_STAT_H */
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c
new file mode 100644
index 000000000000..f8118d39ca9b
--- /dev/null
+++ b/kernel/trace/trace_workqueue.c
@@ -0,0 +1,287 @@
1/*
2 * Workqueue statistical tracer.
3 *
4 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
5 *
6 */
7
8
9#include <trace/workqueue.h>
10#include <linux/list.h>
11#include "trace_stat.h"
12#include "trace.h"
13
14
15/* A cpu workqueue thread */
16struct cpu_workqueue_stats {
17 struct list_head list;
18/* Useful to know if we print the cpu headers */
19 bool first_entry;
20 int cpu;
21 pid_t pid;
22/* Can be inserted from interrupt or user context, need to be atomic */
23 atomic_t inserted;
24/*
25 * Don't need to be atomic, works are serialized in a single workqueue thread
26 * on a single CPU.
27 */
28 unsigned int executed;
29};
30
31/* List of workqueue threads on one cpu */
32struct workqueue_global_stats {
33 struct list_head list;
34 spinlock_t lock;
35};
36
37/* Don't need a global lock because allocated before the workqueues, and
38 * never freed.
39 */
40static struct workqueue_global_stats *all_workqueue_stat;
41
42/* Insertion of a work */
43static void
44probe_workqueue_insertion(struct task_struct *wq_thread,
45 struct work_struct *work)
46{
47 int cpu = cpumask_first(&wq_thread->cpus_allowed);
48 struct cpu_workqueue_stats *node, *next;
49 unsigned long flags;
50
51 spin_lock_irqsave(&all_workqueue_stat[cpu].lock, flags);
52 list_for_each_entry_safe(node, next, &all_workqueue_stat[cpu].list,
53 list) {
54 if (node->pid == wq_thread->pid) {
55 atomic_inc(&node->inserted);
56 goto found;
57 }
58 }
59 pr_debug("trace_workqueue: entry not found\n");
60found:
61 spin_unlock_irqrestore(&all_workqueue_stat[cpu].lock, flags);
62}
63
64/* Execution of a work */
65static void
66probe_workqueue_execution(struct task_struct *wq_thread,
67 struct work_struct *work)
68{
69 int cpu = cpumask_first(&wq_thread->cpus_allowed);
70 struct cpu_workqueue_stats *node, *next;
71 unsigned long flags;
72
73 spin_lock_irqsave(&all_workqueue_stat[cpu].lock, flags);
74 list_for_each_entry_safe(node, next, &all_workqueue_stat[cpu].list,
75 list) {
76 if (node->pid == wq_thread->pid) {
77 node->executed++;
78 goto found;
79 }
80 }
81 pr_debug("trace_workqueue: entry not found\n");
82found:
83 spin_unlock_irqrestore(&all_workqueue_stat[cpu].lock, flags);
84}
85
86/* Creation of a cpu workqueue thread */
87static void probe_workqueue_creation(struct task_struct *wq_thread, int cpu)
88{
89 struct cpu_workqueue_stats *cws;
90 unsigned long flags;
91
92 WARN_ON(cpu < 0 || cpu >= num_possible_cpus());
93
94 /* Workqueues are sometimes created in atomic context */
95 cws = kzalloc(sizeof(struct cpu_workqueue_stats), GFP_ATOMIC);
96 if (!cws) {
97 pr_warning("trace_workqueue: not enough memory\n");
98 return;
99 }
100 tracing_record_cmdline(wq_thread);
101
102 INIT_LIST_HEAD(&cws->list);
103 cws->cpu = cpu;
104
105 cws->pid = wq_thread->pid;
106
107 spin_lock_irqsave(&all_workqueue_stat[cpu].lock, flags);
108 if (list_empty(&all_workqueue_stat[cpu].list))
109 cws->first_entry = true;
110 list_add_tail(&cws->list, &all_workqueue_stat[cpu].list);
111 spin_unlock_irqrestore(&all_workqueue_stat[cpu].lock, flags);
112}
113
114/* Destruction of a cpu workqueue thread */
115static void probe_workqueue_destruction(struct task_struct *wq_thread)
116{
117 /* Workqueue only execute on one cpu */
118 int cpu = cpumask_first(&wq_thread->cpus_allowed);
119 struct cpu_workqueue_stats *node, *next;
120 unsigned long flags;
121
122 spin_lock_irqsave(&all_workqueue_stat[cpu].lock, flags);
123 list_for_each_entry_safe(node, next, &all_workqueue_stat[cpu].list,
124 list) {
125 if (node->pid == wq_thread->pid) {
126 list_del(&node->list);
127 kfree(node);
128 goto found;
129 }
130 }
131
132 pr_debug("trace_workqueue: don't find workqueue to destroy\n");
133found:
134 spin_unlock_irqrestore(&all_workqueue_stat[cpu].lock, flags);
135
136}
137
138static struct cpu_workqueue_stats *workqueue_stat_start_cpu(int cpu)
139{
140 unsigned long flags;
141 struct cpu_workqueue_stats *ret = NULL;
142
143
144 spin_lock_irqsave(&all_workqueue_stat[cpu].lock, flags);
145
146 if (!list_empty(&all_workqueue_stat[cpu].list))
147 ret = list_entry(all_workqueue_stat[cpu].list.next,
148 struct cpu_workqueue_stats, list);
149
150 spin_unlock_irqrestore(&all_workqueue_stat[cpu].lock, flags);
151
152 return ret;
153}
154
155static void *workqueue_stat_start(void)
156{
157 int cpu;
158 void *ret = NULL;
159
160 for_each_possible_cpu(cpu) {
161 ret = workqueue_stat_start_cpu(cpu);
162 if (ret)
163 return ret;
164 }
165 return NULL;
166}
167
168static void *workqueue_stat_next(void *prev, int idx)
169{
170 struct cpu_workqueue_stats *prev_cws = prev;
171 int cpu = prev_cws->cpu;
172 unsigned long flags;
173 void *ret = NULL;
174
175 spin_lock_irqsave(&all_workqueue_stat[cpu].lock, flags);
176 if (list_is_last(&prev_cws->list, &all_workqueue_stat[cpu].list)) {
177 spin_unlock_irqrestore(&all_workqueue_stat[cpu].lock, flags);
178 for (++cpu ; cpu < num_possible_cpus(); cpu++) {
179 ret = workqueue_stat_start_cpu(cpu);
180 if (ret)
181 return ret;
182 }
183 return NULL;
184 }
185 spin_unlock_irqrestore(&all_workqueue_stat[cpu].lock, flags);
186
187 return list_entry(prev_cws->list.next, struct cpu_workqueue_stats,
188 list);
189}
190
191static int workqueue_stat_show(struct seq_file *s, void *p)
192{
193 struct cpu_workqueue_stats *cws = p;
194 unsigned long flags;
195 int cpu = cws->cpu;
196
197 seq_printf(s, "%3d %6d %6u %s\n", cws->cpu,
198 atomic_read(&cws->inserted),
199 cws->executed,
200 trace_find_cmdline(cws->pid));
201
202 spin_lock_irqsave(&all_workqueue_stat[cpu].lock, flags);
203 if (&cws->list == all_workqueue_stat[cpu].list.next)
204 seq_printf(s, "\n");
205 spin_unlock_irqrestore(&all_workqueue_stat[cpu].lock, flags);
206
207 return 0;
208}
209
210static int workqueue_stat_headers(struct seq_file *s)
211{
212 seq_printf(s, "# CPU INSERTED EXECUTED NAME\n");
213 seq_printf(s, "# | | | |\n\n");
214 return 0;
215}
216
217struct tracer_stat workqueue_stats __read_mostly = {
218 .name = "workqueues",
219 .stat_start = workqueue_stat_start,
220 .stat_next = workqueue_stat_next,
221 .stat_show = workqueue_stat_show,
222 .stat_headers = workqueue_stat_headers
223};
224
225
226int __init stat_workqueue_init(void)
227{
228 if (register_stat_tracer(&workqueue_stats)) {
229 pr_warning("Unable to register workqueue stat tracer\n");
230 return 1;
231 }
232
233 return 0;
234}
235fs_initcall(stat_workqueue_init);
236
237/*
238 * Workqueues are created very early, just after pre-smp initcalls.
239 * So we must register our tracepoints at this stage.
240 */
241int __init trace_workqueue_early_init(void)
242{
243 int ret, cpu;
244
245 ret = register_trace_workqueue_insertion(probe_workqueue_insertion);
246 if (ret)
247 goto out;
248
249 ret = register_trace_workqueue_execution(probe_workqueue_execution);
250 if (ret)
251 goto no_insertion;
252
253 ret = register_trace_workqueue_creation(probe_workqueue_creation);
254 if (ret)
255 goto no_execution;
256
257 ret = register_trace_workqueue_destruction(probe_workqueue_destruction);
258 if (ret)
259 goto no_creation;
260
261 all_workqueue_stat = kmalloc(sizeof(struct workqueue_global_stats)
262 * num_possible_cpus(), GFP_KERNEL);
263
264 if (!all_workqueue_stat) {
265 pr_warning("trace_workqueue: not enough memory\n");
266 goto no_creation;
267 }
268
269 for_each_possible_cpu(cpu) {
270 spin_lock_init(&all_workqueue_stat[cpu].lock);
271 INIT_LIST_HEAD(&all_workqueue_stat[cpu].list);
272 }
273
274 return 0;
275
276no_creation:
277 unregister_trace_workqueue_creation(probe_workqueue_creation);
278no_execution:
279 unregister_trace_workqueue_execution(probe_workqueue_execution);
280no_insertion:
281 unregister_trace_workqueue_insertion(probe_workqueue_insertion);
282out:
283 pr_warning("trace_workqueue: unable to trace workqueues\n");
284
285 return 1;
286}
287early_initcall(trace_workqueue_early_init);
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 2f445833ae37..1fc2bc20603f 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -33,6 +33,7 @@
33#include <linux/kallsyms.h> 33#include <linux/kallsyms.h>
34#include <linux/debug_locks.h> 34#include <linux/debug_locks.h>
35#include <linux/lockdep.h> 35#include <linux/lockdep.h>
36#include <trace/workqueue.h>
36 37
37/* 38/*
38 * The per-CPU workqueue (if single thread, we always use the first 39 * The per-CPU workqueue (if single thread, we always use the first
@@ -125,9 +126,13 @@ struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
125 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK); 126 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
126} 127}
127 128
129DEFINE_TRACE(workqueue_insertion);
130
128static void insert_work(struct cpu_workqueue_struct *cwq, 131static void insert_work(struct cpu_workqueue_struct *cwq,
129 struct work_struct *work, struct list_head *head) 132 struct work_struct *work, struct list_head *head)
130{ 133{
134 trace_workqueue_insertion(cwq->thread, work);
135
131 set_wq_data(work, cwq); 136 set_wq_data(work, cwq);
132 /* 137 /*
133 * Ensure that we get the right work->data if we see the 138 * Ensure that we get the right work->data if we see the
@@ -259,6 +264,8 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
259} 264}
260EXPORT_SYMBOL_GPL(queue_delayed_work_on); 265EXPORT_SYMBOL_GPL(queue_delayed_work_on);
261 266
267DEFINE_TRACE(workqueue_execution);
268
262static void run_workqueue(struct cpu_workqueue_struct *cwq) 269static void run_workqueue(struct cpu_workqueue_struct *cwq)
263{ 270{
264 spin_lock_irq(&cwq->lock); 271 spin_lock_irq(&cwq->lock);
@@ -284,7 +291,7 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq)
284 */ 291 */
285 struct lockdep_map lockdep_map = work->lockdep_map; 292 struct lockdep_map lockdep_map = work->lockdep_map;
286#endif 293#endif
287 294 trace_workqueue_execution(cwq->thread, work);
288 cwq->current_work = work; 295 cwq->current_work = work;
289 list_del_init(cwq->worklist.next); 296 list_del_init(cwq->worklist.next);
290 spin_unlock_irq(&cwq->lock); 297 spin_unlock_irq(&cwq->lock);
@@ -765,6 +772,8 @@ init_cpu_workqueue(struct workqueue_struct *wq, int cpu)
765 return cwq; 772 return cwq;
766} 773}
767 774
775DEFINE_TRACE(workqueue_creation);
776
768static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) 777static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
769{ 778{
770 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 779 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
@@ -787,6 +796,8 @@ static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
787 sched_setscheduler_nocheck(p, SCHED_FIFO, &param); 796 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
788 cwq->thread = p; 797 cwq->thread = p;
789 798
799 trace_workqueue_creation(cwq->thread, cpu);
800
790 return 0; 801 return 0;
791} 802}
792 803
@@ -868,6 +879,8 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
868} 879}
869EXPORT_SYMBOL_GPL(__create_workqueue_key); 880EXPORT_SYMBOL_GPL(__create_workqueue_key);
870 881
882DEFINE_TRACE(workqueue_destruction);
883
871static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq) 884static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
872{ 885{
873 /* 886 /*
@@ -891,6 +904,7 @@ static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
891 * checks list_empty(), and a "normal" queue_work() can't use 904 * checks list_empty(), and a "normal" queue_work() can't use
892 * a dead CPU. 905 * a dead CPU.
893 */ 906 */
907 trace_workqueue_destruction(cwq->thread);
894 kthread_stop(cwq->thread); 908 kthread_stop(cwq->thread);
895 cwq->thread = NULL; 909 cwq->thread = NULL;
896} 910}
diff --git a/mm/slab.c b/mm/slab.c
index ddc41f337d58..dae716b32915 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -102,6 +102,7 @@
102#include <linux/cpu.h> 102#include <linux/cpu.h>
103#include <linux/sysctl.h> 103#include <linux/sysctl.h>
104#include <linux/module.h> 104#include <linux/module.h>
105#include <trace/kmemtrace.h>
105#include <linux/rcupdate.h> 106#include <linux/rcupdate.h>
106#include <linux/string.h> 107#include <linux/string.h>
107#include <linux/uaccess.h> 108#include <linux/uaccess.h>
@@ -568,6 +569,14 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp)
568 569
569#endif 570#endif
570 571
572#ifdef CONFIG_KMEMTRACE
573size_t slab_buffer_size(struct kmem_cache *cachep)
574{
575 return cachep->buffer_size;
576}
577EXPORT_SYMBOL(slab_buffer_size);
578#endif
579
571/* 580/*
572 * Do not go above this order unless 0 objects fit into the slab. 581 * Do not go above this order unless 0 objects fit into the slab.
573 */ 582 */
@@ -3550,10 +3559,23 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
3550 */ 3559 */
3551void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) 3560void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3552{ 3561{
3553 return __cache_alloc(cachep, flags, __builtin_return_address(0)); 3562 void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
3563
3564 kmemtrace_mark_alloc(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
3565 obj_size(cachep), cachep->buffer_size, flags);
3566
3567 return ret;
3554} 3568}
3555EXPORT_SYMBOL(kmem_cache_alloc); 3569EXPORT_SYMBOL(kmem_cache_alloc);
3556 3570
3571#ifdef CONFIG_KMEMTRACE
3572void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
3573{
3574 return __cache_alloc(cachep, flags, __builtin_return_address(0));
3575}
3576EXPORT_SYMBOL(kmem_cache_alloc_notrace);
3577#endif
3578
3557/** 3579/**
3558 * kmem_ptr_validate - check if an untrusted pointer might be a slab entry. 3580 * kmem_ptr_validate - check if an untrusted pointer might be a slab entry.
3559 * @cachep: the cache we're checking against 3581 * @cachep: the cache we're checking against
@@ -3598,23 +3620,47 @@ out:
3598#ifdef CONFIG_NUMA 3620#ifdef CONFIG_NUMA
3599void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) 3621void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
3600{ 3622{
3601 return __cache_alloc_node(cachep, flags, nodeid, 3623 void *ret = __cache_alloc_node(cachep, flags, nodeid,
3602 __builtin_return_address(0)); 3624 __builtin_return_address(0));
3625
3626 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
3627 obj_size(cachep), cachep->buffer_size,
3628 flags, nodeid);
3629
3630 return ret;
3603} 3631}
3604EXPORT_SYMBOL(kmem_cache_alloc_node); 3632EXPORT_SYMBOL(kmem_cache_alloc_node);
3605 3633
3634#ifdef CONFIG_KMEMTRACE
3635void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
3636 gfp_t flags,
3637 int nodeid)
3638{
3639 return __cache_alloc_node(cachep, flags, nodeid,
3640 __builtin_return_address(0));
3641}
3642EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
3643#endif
3644
3606static __always_inline void * 3645static __always_inline void *
3607__do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller) 3646__do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller)
3608{ 3647{
3609 struct kmem_cache *cachep; 3648 struct kmem_cache *cachep;
3649 void *ret;
3610 3650
3611 cachep = kmem_find_general_cachep(size, flags); 3651 cachep = kmem_find_general_cachep(size, flags);
3612 if (unlikely(ZERO_OR_NULL_PTR(cachep))) 3652 if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3613 return cachep; 3653 return cachep;
3614 return kmem_cache_alloc_node(cachep, flags, node); 3654 ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
3655
3656 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
3657 (unsigned long) caller, ret,
3658 size, cachep->buffer_size, flags, node);
3659
3660 return ret;
3615} 3661}
3616 3662
3617#ifdef CONFIG_DEBUG_SLAB 3663#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
3618void *__kmalloc_node(size_t size, gfp_t flags, int node) 3664void *__kmalloc_node(size_t size, gfp_t flags, int node)
3619{ 3665{
3620 return __do_kmalloc_node(size, flags, node, 3666 return __do_kmalloc_node(size, flags, node,
@@ -3647,6 +3693,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3647 void *caller) 3693 void *caller)
3648{ 3694{
3649 struct kmem_cache *cachep; 3695 struct kmem_cache *cachep;
3696 void *ret;
3650 3697
3651 /* If you want to save a few bytes .text space: replace 3698 /* If you want to save a few bytes .text space: replace
3652 * __ with kmem_. 3699 * __ with kmem_.
@@ -3656,11 +3703,17 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3656 cachep = __find_general_cachep(size, flags); 3703 cachep = __find_general_cachep(size, flags);
3657 if (unlikely(ZERO_OR_NULL_PTR(cachep))) 3704 if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3658 return cachep; 3705 return cachep;
3659 return __cache_alloc(cachep, flags, caller); 3706 ret = __cache_alloc(cachep, flags, caller);
3707
3708 kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC,
3709 (unsigned long) caller, ret,
3710 size, cachep->buffer_size, flags);
3711
3712 return ret;
3660} 3713}
3661 3714
3662 3715
3663#ifdef CONFIG_DEBUG_SLAB 3716#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
3664void *__kmalloc(size_t size, gfp_t flags) 3717void *__kmalloc(size_t size, gfp_t flags)
3665{ 3718{
3666 return __do_kmalloc(size, flags, __builtin_return_address(0)); 3719 return __do_kmalloc(size, flags, __builtin_return_address(0));
@@ -3699,6 +3752,8 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
3699 debug_check_no_obj_freed(objp, obj_size(cachep)); 3752 debug_check_no_obj_freed(objp, obj_size(cachep));
3700 __cache_free(cachep, objp); 3753 __cache_free(cachep, objp);
3701 local_irq_restore(flags); 3754 local_irq_restore(flags);
3755
3756 kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, objp);
3702} 3757}
3703EXPORT_SYMBOL(kmem_cache_free); 3758EXPORT_SYMBOL(kmem_cache_free);
3704 3759
@@ -3725,6 +3780,8 @@ void kfree(const void *objp)
3725 debug_check_no_obj_freed(objp, obj_size(c)); 3780 debug_check_no_obj_freed(objp, obj_size(c));
3726 __cache_free(c, (void *)objp); 3781 __cache_free(c, (void *)objp);
3727 local_irq_restore(flags); 3782 local_irq_restore(flags);
3783
3784 kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, objp);
3728} 3785}
3729EXPORT_SYMBOL(kfree); 3786EXPORT_SYMBOL(kfree);
3730 3787
diff --git a/mm/slob.c b/mm/slob.c
index bf7e8fc3aed8..4d1c0fc33b6b 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -65,6 +65,7 @@
65#include <linux/module.h> 65#include <linux/module.h>
66#include <linux/rcupdate.h> 66#include <linux/rcupdate.h>
67#include <linux/list.h> 67#include <linux/list.h>
68#include <trace/kmemtrace.h>
68#include <asm/atomic.h> 69#include <asm/atomic.h>
69 70
70/* 71/*
@@ -463,27 +464,38 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
463{ 464{
464 unsigned int *m; 465 unsigned int *m;
465 int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); 466 int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
467 void *ret;
466 468
467 if (size < PAGE_SIZE - align) { 469 if (size < PAGE_SIZE - align) {
468 if (!size) 470 if (!size)
469 return ZERO_SIZE_PTR; 471 return ZERO_SIZE_PTR;
470 472
471 m = slob_alloc(size + align, gfp, align, node); 473 m = slob_alloc(size + align, gfp, align, node);
474
472 if (!m) 475 if (!m)
473 return NULL; 476 return NULL;
474 *m = size; 477 *m = size;
475 return (void *)m + align; 478 ret = (void *)m + align;
479
480 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
481 _RET_IP_, ret,
482 size, size + align, gfp, node);
476 } else { 483 } else {
477 void *ret; 484 unsigned int order = get_order(size);
478 485
479 ret = slob_new_page(gfp | __GFP_COMP, get_order(size), node); 486 ret = slob_new_page(gfp | __GFP_COMP, order, node);
480 if (ret) { 487 if (ret) {
481 struct page *page; 488 struct page *page;
482 page = virt_to_page(ret); 489 page = virt_to_page(ret);
483 page->private = size; 490 page->private = size;
484 } 491 }
485 return ret; 492
493 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
494 _RET_IP_, ret,
495 size, PAGE_SIZE << order, gfp, node);
486 } 496 }
497
498 return ret;
487} 499}
488EXPORT_SYMBOL(__kmalloc_node); 500EXPORT_SYMBOL(__kmalloc_node);
489 501
@@ -501,6 +513,8 @@ void kfree(const void *block)
501 slob_free(m, *m + align); 513 slob_free(m, *m + align);
502 } else 514 } else
503 put_page(&sp->page); 515 put_page(&sp->page);
516
517 kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, block);
504} 518}
505EXPORT_SYMBOL(kfree); 519EXPORT_SYMBOL(kfree);
506 520
@@ -569,10 +583,19 @@ void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
569{ 583{
570 void *b; 584 void *b;
571 585
572 if (c->size < PAGE_SIZE) 586 if (c->size < PAGE_SIZE) {
573 b = slob_alloc(c->size, flags, c->align, node); 587 b = slob_alloc(c->size, flags, c->align, node);
574 else 588 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE,
589 _RET_IP_, b, c->size,
590 SLOB_UNITS(c->size) * SLOB_UNIT,
591 flags, node);
592 } else {
575 b = slob_new_page(flags, get_order(c->size), node); 593 b = slob_new_page(flags, get_order(c->size), node);
594 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE,
595 _RET_IP_, b, c->size,
596 PAGE_SIZE << get_order(c->size),
597 flags, node);
598 }
576 599
577 if (c->ctor) 600 if (c->ctor)
578 c->ctor(b); 601 c->ctor(b);
@@ -608,6 +631,8 @@ void kmem_cache_free(struct kmem_cache *c, void *b)
608 } else { 631 } else {
609 __kmem_cache_free(b, c->size); 632 __kmem_cache_free(b, c->size);
610 } 633 }
634
635 kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, b);
611} 636}
612EXPORT_SYMBOL(kmem_cache_free); 637EXPORT_SYMBOL(kmem_cache_free);
613 638
diff --git a/mm/slub.c b/mm/slub.c
index 6392ae5cc6b1..f657c88814ee 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -16,6 +16,7 @@
16#include <linux/slab.h> 16#include <linux/slab.h>
17#include <linux/proc_fs.h> 17#include <linux/proc_fs.h>
18#include <linux/seq_file.h> 18#include <linux/seq_file.h>
19#include <trace/kmemtrace.h>
19#include <linux/cpu.h> 20#include <linux/cpu.h>
20#include <linux/cpuset.h> 21#include <linux/cpuset.h>
21#include <linux/mempolicy.h> 22#include <linux/mempolicy.h>
@@ -1623,18 +1624,46 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
1623 1624
1624void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) 1625void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
1625{ 1626{
1626 return slab_alloc(s, gfpflags, -1, _RET_IP_); 1627 void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_);
1628
1629 kmemtrace_mark_alloc(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
1630 s->objsize, s->size, gfpflags);
1631
1632 return ret;
1627} 1633}
1628EXPORT_SYMBOL(kmem_cache_alloc); 1634EXPORT_SYMBOL(kmem_cache_alloc);
1629 1635
1636#ifdef CONFIG_KMEMTRACE
1637void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
1638{
1639 return slab_alloc(s, gfpflags, -1, _RET_IP_);
1640}
1641EXPORT_SYMBOL(kmem_cache_alloc_notrace);
1642#endif
1643
1630#ifdef CONFIG_NUMA 1644#ifdef CONFIG_NUMA
1631void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) 1645void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
1632{ 1646{
1633 return slab_alloc(s, gfpflags, node, _RET_IP_); 1647 void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
1648
1649 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
1650 s->objsize, s->size, gfpflags, node);
1651
1652 return ret;
1634} 1653}
1635EXPORT_SYMBOL(kmem_cache_alloc_node); 1654EXPORT_SYMBOL(kmem_cache_alloc_node);
1636#endif 1655#endif
1637 1656
1657#ifdef CONFIG_KMEMTRACE
1658void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
1659 gfp_t gfpflags,
1660 int node)
1661{
1662 return slab_alloc(s, gfpflags, node, _RET_IP_);
1663}
1664EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
1665#endif
1666
1638/* 1667/*
1639 * Slow patch handling. This may still be called frequently since objects 1668 * Slow patch handling. This may still be called frequently since objects
1640 * have a longer lifetime than the cpu slabs in most processing loads. 1669 * have a longer lifetime than the cpu slabs in most processing loads.
@@ -1742,6 +1771,8 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
1742 page = virt_to_head_page(x); 1771 page = virt_to_head_page(x);
1743 1772
1744 slab_free(s, page, x, _RET_IP_); 1773 slab_free(s, page, x, _RET_IP_);
1774
1775 kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, x);
1745} 1776}
1746EXPORT_SYMBOL(kmem_cache_free); 1777EXPORT_SYMBOL(kmem_cache_free);
1747 1778
@@ -2657,6 +2688,7 @@ static struct kmem_cache *get_slab(size_t size, gfp_t flags)
2657void *__kmalloc(size_t size, gfp_t flags) 2688void *__kmalloc(size_t size, gfp_t flags)
2658{ 2689{
2659 struct kmem_cache *s; 2690 struct kmem_cache *s;
2691 void *ret;
2660 2692
2661 if (unlikely(size > PAGE_SIZE)) 2693 if (unlikely(size > PAGE_SIZE))
2662 return kmalloc_large(size, flags); 2694 return kmalloc_large(size, flags);
@@ -2666,7 +2698,12 @@ void *__kmalloc(size_t size, gfp_t flags)
2666 if (unlikely(ZERO_OR_NULL_PTR(s))) 2698 if (unlikely(ZERO_OR_NULL_PTR(s)))
2667 return s; 2699 return s;
2668 2700
2669 return slab_alloc(s, flags, -1, _RET_IP_); 2701 ret = slab_alloc(s, flags, -1, _RET_IP_);
2702
2703 kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, ret,
2704 size, s->size, flags);
2705
2706 return ret;
2670} 2707}
2671EXPORT_SYMBOL(__kmalloc); 2708EXPORT_SYMBOL(__kmalloc);
2672 2709
@@ -2685,16 +2722,30 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
2685void *__kmalloc_node(size_t size, gfp_t flags, int node) 2722void *__kmalloc_node(size_t size, gfp_t flags, int node)
2686{ 2723{
2687 struct kmem_cache *s; 2724 struct kmem_cache *s;
2725 void *ret;
2688 2726
2689 if (unlikely(size > PAGE_SIZE)) 2727 if (unlikely(size > PAGE_SIZE)) {
2690 return kmalloc_large_node(size, flags, node); 2728 ret = kmalloc_large_node(size, flags, node);
2729
2730 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
2731 _RET_IP_, ret,
2732 size, PAGE_SIZE << get_order(size),
2733 flags, node);
2734
2735 return ret;
2736 }
2691 2737
2692 s = get_slab(size, flags); 2738 s = get_slab(size, flags);
2693 2739
2694 if (unlikely(ZERO_OR_NULL_PTR(s))) 2740 if (unlikely(ZERO_OR_NULL_PTR(s)))
2695 return s; 2741 return s;
2696 2742
2697 return slab_alloc(s, flags, node, _RET_IP_); 2743 ret = slab_alloc(s, flags, node, _RET_IP_);
2744
2745 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, ret,
2746 size, s->size, flags, node);
2747
2748 return ret;
2698} 2749}
2699EXPORT_SYMBOL(__kmalloc_node); 2750EXPORT_SYMBOL(__kmalloc_node);
2700#endif 2751#endif
@@ -2752,6 +2803,8 @@ void kfree(const void *x)
2752 return; 2803 return;
2753 } 2804 }
2754 slab_free(page->slab, page, object, _RET_IP_); 2805 slab_free(page->slab, page, object, _RET_IP_);
2806
2807 kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, x);
2755} 2808}
2756EXPORT_SYMBOL(kfree); 2809EXPORT_SYMBOL(kfree);
2757 2810
@@ -3221,6 +3274,7 @@ static struct notifier_block __cpuinitdata slab_notifier = {
3221void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) 3274void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
3222{ 3275{
3223 struct kmem_cache *s; 3276 struct kmem_cache *s;
3277 void *ret;
3224 3278
3225 if (unlikely(size > PAGE_SIZE)) 3279 if (unlikely(size > PAGE_SIZE))
3226 return kmalloc_large(size, gfpflags); 3280 return kmalloc_large(size, gfpflags);
@@ -3230,13 +3284,20 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
3230 if (unlikely(ZERO_OR_NULL_PTR(s))) 3284 if (unlikely(ZERO_OR_NULL_PTR(s)))
3231 return s; 3285 return s;
3232 3286
3233 return slab_alloc(s, gfpflags, -1, caller); 3287 ret = slab_alloc(s, gfpflags, -1, caller);
3288
3289 /* Honor the call site pointer we recieved. */
3290 kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, caller, ret, size,
3291 s->size, gfpflags);
3292
3293 return ret;
3234} 3294}
3235 3295
3236void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, 3296void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
3237 int node, unsigned long caller) 3297 int node, unsigned long caller)
3238{ 3298{
3239 struct kmem_cache *s; 3299 struct kmem_cache *s;
3300 void *ret;
3240 3301
3241 if (unlikely(size > PAGE_SIZE)) 3302 if (unlikely(size > PAGE_SIZE))
3242 return kmalloc_large_node(size, gfpflags, node); 3303 return kmalloc_large_node(size, gfpflags, node);
@@ -3246,7 +3307,13 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
3246 if (unlikely(ZERO_OR_NULL_PTR(s))) 3307 if (unlikely(ZERO_OR_NULL_PTR(s)))
3247 return s; 3308 return s;
3248 3309
3249 return slab_alloc(s, gfpflags, node, caller); 3310 ret = slab_alloc(s, gfpflags, node, caller);
3311
3312 /* Honor the call site pointer we recieved. */
3313 kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, caller, ret,
3314 size, s->size, gfpflags, node);
3315
3316 return ret;
3250} 3317}
3251 3318
3252#ifdef CONFIG_SLUB_DEBUG 3319#ifdef CONFIG_SLUB_DEBUG
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index 5d900307de3e..b5efa98994bb 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -112,13 +112,13 @@ endif
112# --------------------------------------------------------------------------- 112# ---------------------------------------------------------------------------
113 113
114# Default is built-in, unless we know otherwise 114# Default is built-in, unless we know otherwise
115modkern_cflags := $(CFLAGS_KERNEL) 115modkern_cflags = $(if $(part-of-module), $(CFLAGS_MODULE), $(CFLAGS_KERNEL))
116quiet_modtag := $(empty) $(empty) 116quiet_modtag := $(empty) $(empty)
117 117
118$(real-objs-m) : modkern_cflags := $(CFLAGS_MODULE) 118$(real-objs-m) : part-of-module := y
119$(real-objs-m:.o=.i) : modkern_cflags := $(CFLAGS_MODULE) 119$(real-objs-m:.o=.i) : part-of-module := y
120$(real-objs-m:.o=.s) : modkern_cflags := $(CFLAGS_MODULE) 120$(real-objs-m:.o=.s) : part-of-module := y
121$(real-objs-m:.o=.lst): modkern_cflags := $(CFLAGS_MODULE) 121$(real-objs-m:.o=.lst): part-of-module := y
122 122
123$(real-objs-m) : quiet_modtag := [M] 123$(real-objs-m) : quiet_modtag := [M]
124$(real-objs-m:.o=.i) : quiet_modtag := [M] 124$(real-objs-m:.o=.i) : quiet_modtag := [M]
@@ -215,7 +215,8 @@ endif
215ifdef CONFIG_FTRACE_MCOUNT_RECORD 215ifdef CONFIG_FTRACE_MCOUNT_RECORD
216cmd_record_mcount = perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \ 216cmd_record_mcount = perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \
217 "$(if $(CONFIG_64BIT),64,32)" \ 217 "$(if $(CONFIG_64BIT),64,32)" \
218 "$(OBJDUMP)" "$(OBJCOPY)" "$(CC)" "$(LD)" "$(NM)" "$(RM)" "$(MV)" "$(@)"; 218 "$(OBJDUMP)" "$(OBJCOPY)" "$(CC)" "$(LD)" "$(NM)" "$(RM)" "$(MV)" \
219 "$(if $(part-of-module),1,0)" "$(@)";
219endif 220endif
220 221
221define rule_cc_o_c 222define rule_cc_o_c
diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl
index fe831412bea9..2ded5c8e6b85 100755
--- a/scripts/recordmcount.pl
+++ b/scripts/recordmcount.pl
@@ -100,14 +100,19 @@ $P =~ s@.*/@@g;
100 100
101my $V = '0.1'; 101my $V = '0.1';
102 102
103if ($#ARGV < 6) { 103if ($#ARGV < 7) {
104 print "usage: $P arch objdump objcopy cc ld nm rm mv inputfile\n"; 104 print "usage: $P arch objdump objcopy cc ld nm rm mv is_module inputfile\n";
105 print "version: $V\n"; 105 print "version: $V\n";
106 exit(1); 106 exit(1);
107} 107}
108 108
109my ($arch, $bits, $objdump, $objcopy, $cc, 109my ($arch, $bits, $objdump, $objcopy, $cc,
110 $ld, $nm, $rm, $mv, $inputfile) = @ARGV; 110 $ld, $nm, $rm, $mv, $is_module, $inputfile) = @ARGV;
111
112# This file refers to mcount and shouldn't be ftraced, so lets' ignore it
113if ($inputfile eq "kernel/trace/ftrace.o") {
114 exit(0);
115}
111 116
112# Acceptable sections to record. 117# Acceptable sections to record.
113my %text_sections = ( 118my %text_sections = (
@@ -201,6 +206,13 @@ if ($arch eq "x86_64") {
201 $alignment = 2; 206 $alignment = 2;
202 $section_type = '%progbits'; 207 $section_type = '%progbits';
203 208
209} elsif ($arch eq "ia64") {
210 $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\s_mcount\$";
211 $type = "data8";
212
213 if ($is_module eq "0") {
214 $cc .= " -mconstant-gp";
215 }
204} else { 216} else {
205 die "Arch $arch is not supported with CONFIG_FTRACE_MCOUNT_RECORD"; 217 die "Arch $arch is not supported with CONFIG_FTRACE_MCOUNT_RECORD";
206} 218}
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-23 21:10:22 -0500