tracing/function-return-tracer: set a more human readable output

Impact: feature

This patch sets a C-like output for the function graph tracing.
For this aim, we now call two handler for each function: one on the entry
and one other on return. This way we can draw a well-ordered call stack.

The pid of the previous trace is loosely stored to be compared against
the one of the current trace to see if there were a context switch.

Without this little feature, the call tree would seem broken at
some locations.
We could use the sched_tracer to capture these sched_events but this
way of processing is much more simpler.

2 spaces have been chosen for indentation to fit the screen while deep
calls. The time of execution in nanosecs is printed just after closed
braces, it seems more easy this way to find the corresponding function.
If the time was printed as a first column, it would be not so easy to
find the corresponding function if it is called on a deep depth.

I plan to output the return value but on 32 bits CPU, the return value
can be 32 or 64, and its difficult to guess on which case we are.
I don't know what would be the better solution on X86-32: only print
eax (low-part) or even edx (high-part).

Actually it's thee same problem when a function return a 8 bits value, the
high part of eax could contain junk values...

Here is an example of trace:

sys_read() {
  fget_light() {
  } 526
  vfs_read() {
    rw_verify_area() {
      security_file_permission() {
        cap_file_permission() {
        } 519
      } 1564
    } 2640
    do_sync_read() {
      pipe_read() {
        __might_sleep() {
        } 511
        pipe_wait() {
          prepare_to_wait() {
          } 760
          deactivate_task() {
            dequeue_task() {
              dequeue_task_fair() {
                dequeue_entity() {
                  update_curr() {
                    update_min_vruntime() {
                    } 504
                  } 1587
                  clear_buddies() {
                  } 512
                  add_cfs_task_weight() {
                  } 519
                  update_min_vruntime() {
                  } 511
                } 5602
                dequeue_entity() {
                  update_curr() {
                    update_min_vruntime() {
                    } 496
                  } 1631
                  clear_buddies() {
                  } 496
                  update_min_vruntime() {
                  } 527
                } 4580
                hrtick_update() {
                  hrtick_start_fair() {
                  } 488
                } 1489
              } 13700
            } 14949
          } 16016
          msecs_to_jiffies() {
          } 496
          put_prev_task_fair() {
          } 504
          pick_next_task_fair() {
          } 489
          pick_next_task_rt() {
          } 496
          pick_next_task_fair() {
          } 489
          pick_next_task_idle() {
          } 489

------------8<---------- thread 4 ------------8<----------

finish_task_switch() {
} 1203
do_softirq() {
  __do_softirq() {
    __local_bh_disable() {
    } 669
    rcu_process_callbacks() {
      __rcu_process_callbacks() {
        cpu_quiet() {
          rcu_start_batch() {
          } 503
        } 1647
      } 3128
      __rcu_process_callbacks() {
      } 542
    } 5362
    _local_bh_enable() {
    } 587
  } 8880
} 9986
kthread_should_stop() {
} 669
deactivate_task() {
  dequeue_task() {
    dequeue_task_fair() {
      dequeue_entity() {
        update_curr() {
          calc_delta_mine() {
          } 511
          update_min_vruntime() {
          } 511
        } 2813

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 80 insertions, 24 deletions

diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index f5bad4624d2b..b6f0cc2a00cb 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -13,6 +13,7 @@
 
 #include "trace.h"
 
+#define TRACE_GRAPH_INDENT      2
 
 #define TRACE_GRAPH_PRINT_OVERRUN       0x1
 static struct tracer_opt trace_opts[] = {
@@ -26,6 +27,8 @@ static struct tracer_flags tracer_flags = {
         .opts = trace_opts
 };
 
+/* pid on the last trace processed */
+static pid_t last_pid = -1;
 
 static int graph_trace_init(struct trace_array *tr)
 {
@@ -33,7 +36,8 @@ static int graph_trace_init(struct trace_array *tr)
         for_each_online_cpu(cpu)
                 tracing_reset(tr, cpu);
 
-        return register_ftrace_graph(&trace_function_graph);
+        return register_ftrace_graph(&trace_graph_return,
+                                        &trace_graph_entry);
 }
 
 static void graph_trace_reset(struct trace_array *tr)
@@ -41,45 +45,97 @@ static void graph_trace_reset(struct trace_array *tr)
                 unregister_ftrace_graph();
 }
 
+/* If the pid changed since the last trace, output this event */
+static int verif_pid(struct trace_seq *s, pid_t pid)
+{
+        if (last_pid != -1 && last_pid == pid)
+                return 1;
 
-enum print_line_t
-print_graph_function(struct trace_iterator *iter)
+        last_pid = pid;
+        return trace_seq_printf(s, "\n------------8<---------- thread %d"
+                                    " ------------8<----------\n\n",
+                                  pid);
+}
+
+static enum print_line_t
+print_graph_entry(struct ftrace_graph_ent *call, struct trace_seq *s,
+                struct trace_entry *ent)
 {
-        struct trace_seq *s = &iter->seq;
-        struct trace_entry *entry = iter->ent;
-        struct ftrace_graph_entry *field;
+        int i;
         int ret;
 
-        if (entry->type == TRACE_FN_RET) {
-                trace_assign_type(field, entry);
-                ret = trace_seq_printf(s, "%pF -> ", (void *)field->parent_ip);
-                if (!ret)
-                        return TRACE_TYPE_PARTIAL_LINE;
+        if (!verif_pid(s, ent->pid))
+                return TRACE_TYPE_PARTIAL_LINE;
 
-                ret = seq_print_ip_sym(s, field->ip,
-                                        trace_flags & TRACE_ITER_SYM_MASK);
+        for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
+                ret = trace_seq_printf(s, " ");
                 if (!ret)
                         return TRACE_TYPE_PARTIAL_LINE;
+        }
+
+        ret = seq_print_ip_sym(s, call->func, 0);
+        if (!ret)
+                return TRACE_TYPE_PARTIAL_LINE;
+
+        ret = trace_seq_printf(s, "() {\n");
+        if (!ret)
+                return TRACE_TYPE_PARTIAL_LINE;
+        return TRACE_TYPE_HANDLED;
+}
+
+static enum print_line_t
+print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
+                   struct trace_entry *ent)
+{
+        int i;
+        int ret;
+
+        if (!verif_pid(s, ent->pid))
+                return TRACE_TYPE_PARTIAL_LINE;
 
-                ret = trace_seq_printf(s, " (%llu ns)",
-                                        field->rettime - field->calltime);
+        for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) {
+                ret = trace_seq_printf(s, " ");
                 if (!ret)
                         return TRACE_TYPE_PARTIAL_LINE;
+        }
+
+        ret = trace_seq_printf(s, "} ");
+        if (!ret)
+                return TRACE_TYPE_PARTIAL_LINE;
 
-                if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERRUN) {
-                        ret = trace_seq_printf(s, " (Overruns: %lu)",
-                                                field->overrun);
-                        if (!ret)
-                                return TRACE_TYPE_PARTIAL_LINE;
-                }
+        ret = trace_seq_printf(s, "%llu\n", trace->rettime - trace->calltime);
+        if (!ret)
+                return TRACE_TYPE_PARTIAL_LINE;
 
-                ret = trace_seq_printf(s, "\n");
+        if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERRUN) {
+                ret = trace_seq_printf(s, " (Overruns: %lu)\n",
+                                        trace->overrun);
                 if (!ret)
                         return TRACE_TYPE_PARTIAL_LINE;
+        }
+        return TRACE_TYPE_HANDLED;
+}
+
+enum print_line_t
+print_graph_function(struct trace_iterator *iter)
+{
+        struct trace_seq *s = &iter->seq;
+        struct trace_entry *entry = iter->ent;
 
-                return TRACE_TYPE_HANDLED;
+        switch (entry->type) {
+        case TRACE_GRAPH_ENT: {
+                struct ftrace_graph_ent_entry *field;
+                trace_assign_type(field, entry);
+                return print_graph_entry(&field->graph_ent, s, entry);
+        }
+        case TRACE_GRAPH_RET: {
+                struct ftrace_graph_ret_entry *field;
+                trace_assign_type(field, entry);
+                return print_graph_return(&field->ret, s, entry);
+        }
+        default:
+                return TRACE_TYPE_UNHANDLED;
         }
-        return TRACE_TYPE_UNHANDLED;
 }
 
 static struct tracer graph_trace __read_mostly = {