tracing: Added hardware latency tracer

The hardware latency tracer has been in the PREEMPT_RT patch for some time.
It is used to detect possible SMIs or any other hardware interruptions that
the kernel is unaware of. Note, NMIs may also be detected, but that may be
good to note as well.

The logic is pretty simple. It simply creates a thread that spins on a
single CPU for a specified amount of time (width) within a periodic window
(window). These numbers may be adjusted by their cooresponding names in

   /sys/kernel/tracing/hwlat_detector/

The defaults are window = 1000000 us (1 second)
                 width  =  500000 us (1/2 second)

The loop consists of:

	t1 = trace_clock_local();
	t2 = trace_clock_local();

Where trace_clock_local() is a variant of sched_clock().

The difference of t2 - t1 is recorded as the "inner" timestamp and also the
timestamp  t1 - prev_t2 is recorded as the "outer" timestamp. If either of
these differences are greater than the time denoted in
/sys/kernel/tracing/tracing_thresh then it records the event.

When this tracer is started, and tracing_thresh is zero, it changes to the
default threshold of 10 us.

The hwlat tracer in the PREEMPT_RT patch was originally written by
Jon Masters. I have modified it quite a bit and turned it into a
tracer.

Based-on-code-by: Jon Masters <jcm@redhat.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

1 files changed, 52 insertions, 0 deletions

diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index 0bb9cf2d53e6..d67a562df259 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -1098,6 +1098,57 @@ static struct trace_event trace_user_stack_event = {
         .funcs          = &trace_user_stack_funcs,
 };
 
+/* TRACE_HWLAT */
+static enum print_line_t
+trace_hwlat_print(struct trace_iterator *iter, int flags,
+                  struct trace_event *event)
+{
+        struct trace_entry *entry = iter->ent;
+        struct trace_seq *s = &iter->seq;
+        struct hwlat_entry *field;
+
+        trace_assign_type(field, entry);
+
+        trace_seq_printf(s, "#%-5u inner/outer(us): %4llu/%-5llu ts:%ld.%09ld\n",
+                         field->seqnum,
+                         field->duration,
+                         field->outer_duration,
+                         field->timestamp.tv_sec,
+                         field->timestamp.tv_nsec);
+
+        return trace_handle_return(s);
+}
+
+
+static enum print_line_t
+trace_hwlat_raw(struct trace_iterator *iter, int flags,
+                struct trace_event *event)
+{
+        struct hwlat_entry *field;
+        struct trace_seq *s = &iter->seq;
+
+        trace_assign_type(field, iter->ent);
+
+        trace_seq_printf(s, "%llu %lld %ld %09ld %u\n",
+                         field->duration,
+                         field->outer_duration,
+                         field->timestamp.tv_sec,
+                         field->timestamp.tv_nsec,
+                         field->seqnum);
+
+        return trace_handle_return(s);
+}
+
+static struct trace_event_functions trace_hwlat_funcs = {
+        .trace          = trace_hwlat_print,
+        .raw            = trace_hwlat_raw,
+};
+
+static struct trace_event trace_hwlat_event = {
+        .type           = TRACE_HWLAT,
+        .funcs          = &trace_hwlat_funcs,
+};
+
 /* TRACE_BPUTS */
 static enum print_line_t
 trace_bputs_print(struct trace_iterator *iter, int flags,
@@ -1233,6 +1284,7 @@ static struct trace_event *events[] __initdata = {
         &trace_bputs_event,
         &trace_bprint_event,
         &trace_print_event,
+        &trace_hwlat_event,
         NULL
 };