perf: 'perf kvm' tool for monitoring guest performance from host

Here is the patch of userspace perf tool.

Signed-off-by: Zhang Yanmin <yanmin_zhang@linux.intel.com>
Signed-off-by: Avi Kivity <avi@redhat.com>

30 files changed, 1407 insertions, 316 deletions

diff --git a/tools/perf/Documentation/perf-kvm.txt b/tools/perf/Documentation/perf-kvm.txt
new file mode 100644
index 000000000000..93400a0f17f0
--- /dev/null
+++ b/tools/perf/Documentation/perf-kvm.txt
@@ -0,0 +1,67 @@
+perf-kvm(1)
+==============
+
+NAME
+----
+perf-kvm - Tool to trace/measure kvm guest os
+
+SYNOPSIS
+--------
+[verse]
+'perf kvm' [--host] [--guest] [--guestmount=<path>
+        [--guestkallsyms=<path> --guestmodules=<path> | --guestvmlinux=<path>]]
+        {top|record|report|diff|buildid-list}
+'perf kvm' [--host] [--guest] [--guestkallsyms=<path> --guestmodules=<path>
+        | --guestvmlinux=<path>] {top|record|report|diff|buildid-list}
+
+DESCRIPTION
+-----------
+There are a couple of variants of perf kvm:
+
+  'perf kvm [options] top <command>' to generates and displays
+  a performance counter profile of guest os in realtime
+  of an arbitrary workload.
+
+  'perf kvm record <command>' to record the performance couinter profile
+  of an arbitrary workload and save it into a perf data file. If both
+  --host and --guest are input, the perf data file name is perf.data.kvm.
+  If there is  no --host but --guest, the file name is perf.data.guest.
+  If there is no --guest but --host, the file name is perf.data.host.
+
+  'perf kvm report' to display the performance counter profile information
+  recorded via perf kvm record.
+
+  'perf kvm diff' to displays the performance difference amongst two perf.data
+  files captured via perf record.
+
+  'perf kvm buildid-list' to  display the buildids found in a perf data file,
+  so that other tools can be used to fetch packages with matching symbol tables
+  for use by perf report.
+
+OPTIONS
+-------
+--host=::
+        Collect host side perforamnce profile.
+--guest=::
+        Collect guest side perforamnce profile.
+--guestmount=<path>::
+        Guest os root file system mount directory. Users mounts guest os
+        root directories under <path> by a specific filesystem access method,
+        typically, sshfs. For example, start 2 guest os. The one's pid is 8888
+        and the other's is 9999.
+        #mkdir ~/guestmount; cd ~/guestmount
+        #sshfs -o allow_other,direct_io -p 5551 localhost:/ 8888/
+        #sshfs -o allow_other,direct_io -p 5552 localhost:/ 9999/
+        #perf kvm --host --guest --guestmount=~/guestmount top
+--guestkallsyms=<path>::
+        Guest os /proc/kallsyms file copy. 'perf' kvm' reads it to get guest
+        kernel symbols. Users copy it out from guest os.
+--guestmodules=<path>::
+        Guest os /proc/modules file copy. 'perf' kvm' reads it to get guest
+        kernel module information. Users copy it out from guest os.
+--guestvmlinux=<path>::
+        Guest os kernel vmlinux.
+
+SEE ALSO
+--------
+linkperf:perf-top[1] perf-record[1] perf-report[1] perf-diff[1] perf-buildid-list[1]
diff --git a/tools/perf/Makefile b/tools/perf/Makefile
index 57b3569716dd..3cb3449a9645 100644
--- a/tools/perf/Makefile
+++ b/tools/perf/Makefile
@@ -472,6 +472,7 @@ BUILTIN_OBJS += $(OUTPUT)builtin-trace.o
 BUILTIN_OBJS += $(OUTPUT)builtin-probe.o
 BUILTIN_OBJS += $(OUTPUT)builtin-kmem.o
 BUILTIN_OBJS += $(OUTPUT)builtin-lock.o
+BUILTIN_OBJS += $(OUTPUT)builtin-kvm.o
 
 PERFLIBS = $(LIB_FILE)
 
diff --git a/tools/perf/builtin-annotate.c b/tools/perf/builtin-annotate.c
index 06eaebe10d04..f924b4332be6 100644
--- a/tools/perf/builtin-annotate.c
+++ b/tools/perf/builtin-annotate.c
@@ -571,7 +571,7 @@ static int __cmd_annotate(void)
                 perf_session__fprintf(session, stdout);
 
         if (verbose > 2)
-                dsos__fprintf(stdout);
+                dsos__fprintf(&session->kerninfo_root, stdout);
 
         perf_session__collapse_resort(&session->hists);
         perf_session__output_resort(&session->hists, session->event_total[0]);
diff --git a/tools/perf/builtin-buildid-list.c b/tools/perf/builtin-buildid-list.c
index af2ad8b92f76..623afe3fdcb8 100644
--- a/tools/perf/builtin-buildid-list.c
+++ b/tools/perf/builtin-buildid-list.c
@@ -46,7 +46,7 @@ static int __cmd_buildid_list(void)
         if (with_hits)
                 perf_session__process_events(session, &build_id__mark_dso_hit_ops);
 
-        dsos__fprintf_buildid(stdout, with_hits);
+        dsos__fprintf_buildid(&session->kerninfo_root, stdout, with_hits);
 
         perf_session__delete(session);
         return err;
diff --git a/tools/perf/builtin-diff.c b/tools/perf/builtin-diff.c
index 3a1d94d75dce..207e860591e2 100644
--- a/tools/perf/builtin-diff.c
+++ b/tools/perf/builtin-diff.c
@@ -33,7 +33,7 @@ static int perf_session__add_hist_entry(struct perf_session *self,
                 return -ENOMEM;
 
         if (hit)
-                he->count += count;
+                __perf_session__add_count(he, al, count);
 
         return 0;
 }
@@ -225,6 +225,10 @@ int cmd_diff(int argc, const char **argv, const char *prefix __used)
                         input_new = argv[1];
                 } else
                         input_new = argv[0];
+        } else if (symbol_conf.default_guest_vmlinux_name ||
+                   symbol_conf.default_guest_kallsyms) {
+                input_old = "perf.data.host";
+                input_new = "perf.data.guest";
         }
 
         symbol_conf.exclude_other = false;
diff --git a/tools/perf/builtin-kmem.c b/tools/perf/builtin-kmem.c
index 513aa8a55db6..db474bbf3322 100644
--- a/tools/perf/builtin-kmem.c
+++ b/tools/perf/builtin-kmem.c
@@ -351,6 +351,7 @@ static void __print_result(struct rb_root *root, struct perf_session *session,
                            int n_lines, int is_caller)
 {
         struct rb_node *next;
+        struct kernel_info *kerninfo;
 
         printf("%.102s\n", graph_dotted_line);
         printf(" %-34s |",  is_caller ? "Callsite": "Alloc Ptr");
@@ -359,10 +360,16 @@ static void __print_result(struct rb_root *root, struct perf_session *session,
 
         next = rb_first(root);
 
+        kerninfo = kerninfo__findhost(&session->kerninfo_root);
+        if (!kerninfo) {
+                pr_err("__print_result: couldn't find kernel information\n");
+                return;
+        }
         while (next && n_lines--) {
                 struct alloc_stat *data = rb_entry(next, struct alloc_stat,
                                                    node);
                 struct symbol *sym = NULL;
+                struct map_groups *kmaps = &kerninfo->kmaps;
                 struct map *map;
                 char buf[BUFSIZ];
                 u64 addr;
@@ -370,8 +377,8 @@ static void __print_result(struct rb_root *root, struct perf_session *session,
                 if (is_caller) {
                         addr = data->call_site;
                         if (!raw_ip)
-                                sym = map_groups__find_function(&session->kmaps,
-                                                                addr, &map, NULL);
+                                sym = map_groups__find_function(kmaps, addr,
+                                                                &map, NULL);
                 } else
                         addr = data->ptr;
 
diff --git a/tools/perf/builtin-kvm.c b/tools/perf/builtin-kvm.c
new file mode 100644
index 000000000000..a4c7cae45024
--- /dev/null
+++ b/tools/perf/builtin-kvm.c
@@ -0,0 +1,144 @@
+#include "builtin.h"
+#include "perf.h"
+
+#include "util/util.h"
+#include "util/cache.h"
+#include "util/symbol.h"
+#include "util/thread.h"
+#include "util/header.h"
+#include "util/session.h"
+
+#include "util/parse-options.h"
+#include "util/trace-event.h"
+
+#include "util/debug.h"
+
+#include <sys/prctl.h>
+
+#include <semaphore.h>
+#include <pthread.h>
+#include <math.h>
+
+static char                     *file_name;
+static char                     name_buffer[256];
+
+int                             perf_host = 1;
+int                             perf_guest;
+
+static const char * const kvm_usage[] = {
+        "perf kvm [<options>] {top|record|report|diff|buildid-list}",
+        NULL
+};
+
+static const struct option kvm_options[] = {
+        OPT_STRING('i', "input", &file_name, "file",
+                   "Input file name"),
+        OPT_STRING('o', "output", &file_name, "file",
+                   "Output file name"),
+        OPT_BOOLEAN(0, "guest", &perf_guest,
+                    "Collect guest os data"),
+        OPT_BOOLEAN(0, "host", &perf_host,
+                    "Collect guest os data"),
+        OPT_STRING(0, "guestmount", &symbol_conf.guestmount, "directory",
+                   "guest mount directory under which every guest os"
+                   " instance has a subdir"),
+        OPT_STRING(0, "guestvmlinux", &symbol_conf.default_guest_vmlinux_name,
+                   "file", "file saving guest os vmlinux"),
+        OPT_STRING(0, "guestkallsyms", &symbol_conf.default_guest_kallsyms,
+                   "file", "file saving guest os /proc/kallsyms"),
+        OPT_STRING(0, "guestmodules", &symbol_conf.default_guest_modules,
+                   "file", "file saving guest os /proc/modules"),
+        OPT_END()
+};
+
+static int __cmd_record(int argc, const char **argv)
+{
+        int rec_argc, i = 0, j;
+        const char **rec_argv;
+
+        rec_argc = argc + 2;
+        rec_argv = calloc(rec_argc + 1, sizeof(char *));
+        rec_argv[i++] = strdup("record");
+        rec_argv[i++] = strdup("-o");
+        rec_argv[i++] = strdup(file_name);
+        for (j = 1; j < argc; j++, i++)
+                rec_argv[i] = argv[j];
+
+        BUG_ON(i != rec_argc);
+
+        return cmd_record(i, rec_argv, NULL);
+}
+
+static int __cmd_report(int argc, const char **argv)
+{
+        int rec_argc, i = 0, j;
+        const char **rec_argv;
+
+        rec_argc = argc + 2;
+        rec_argv = calloc(rec_argc + 1, sizeof(char *));
+        rec_argv[i++] = strdup("report");
+        rec_argv[i++] = strdup("-i");
+        rec_argv[i++] = strdup(file_name);
+        for (j = 1; j < argc; j++, i++)
+                rec_argv[i] = argv[j];
+
+        BUG_ON(i != rec_argc);
+
+        return cmd_report(i, rec_argv, NULL);
+}
+
+static int __cmd_buildid_list(int argc, const char **argv)
+{
+        int rec_argc, i = 0, j;
+        const char **rec_argv;
+
+        rec_argc = argc + 2;
+        rec_argv = calloc(rec_argc + 1, sizeof(char *));
+        rec_argv[i++] = strdup("buildid-list");
+        rec_argv[i++] = strdup("-i");
+        rec_argv[i++] = strdup(file_name);
+        for (j = 1; j < argc; j++, i++)
+                rec_argv[i] = argv[j];
+
+        BUG_ON(i != rec_argc);
+
+        return cmd_buildid_list(i, rec_argv, NULL);
+}
+
+int cmd_kvm(int argc, const char **argv, const char *prefix __used)
+{
+        perf_host = perf_guest = 0;
+
+        argc = parse_options(argc, argv, kvm_options, kvm_usage,
+                        PARSE_OPT_STOP_AT_NON_OPTION);
+        if (!argc)
+                usage_with_options(kvm_usage, kvm_options);
+
+        if (!perf_host)
+                perf_guest = 1;
+
+        if (!file_name) {
+                if (perf_host && !perf_guest)
+                        sprintf(name_buffer, "perf.data.host");
+                else if (!perf_host && perf_guest)
+                        sprintf(name_buffer, "perf.data.guest");
+                else
+                        sprintf(name_buffer, "perf.data.kvm");
+                file_name = name_buffer;
+        }
+
+        if (!strncmp(argv[0], "rec", 3))
+                return __cmd_record(argc, argv);
+        else if (!strncmp(argv[0], "rep", 3))
+                return __cmd_report(argc, argv);
+        else if (!strncmp(argv[0], "diff", 4))
+                return cmd_diff(argc, argv, NULL);
+        else if (!strncmp(argv[0], "top", 3))
+                return cmd_top(argc, argv, NULL);
+        else if (!strncmp(argv[0], "buildid-list", 12))
+                return __cmd_buildid_list(argc, argv);
+        else
+                usage_with_options(kvm_usage, kvm_options);
+
+        return 0;
+}
diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
index a1b99eeac3c0..27f992aca8b5 100644
--- a/tools/perf/builtin-record.c
+++ b/tools/perf/builtin-record.c
@@ -456,6 +456,52 @@ static void atexit_header(void)
         }
 }
 
+static void event__synthesize_guest_os(struct kernel_info *kerninfo,
+                void *data __attribute__((unused)))
+{
+        int err;
+        char *guest_kallsyms;
+        char path[PATH_MAX];
+
+        if (is_host_kernel(kerninfo))
+                return;
+
+        /*
+         *As for guest kernel when processing subcommand record&report,
+         *we arrange module mmap prior to guest kernel mmap and trigger
+         *a preload dso because default guest module symbols are loaded
+         *from guest kallsyms instead of /lib/modules/XXX/XXX. This
+         *method is used to avoid symbol missing when the first addr is
+         *in module instead of in guest kernel.
+         */
+        err = event__synthesize_modules(process_synthesized_event,
+                        session,
+                        kerninfo);
+        if (err < 0)
+                pr_err("Couldn't record guest kernel [%d]'s reference"
+                        " relocation symbol.\n", kerninfo->pid);
+
+        if (is_default_guest(kerninfo))
+                guest_kallsyms = (char *) symbol_conf.default_guest_kallsyms;
+        else {
+                sprintf(path, "%s/proc/kallsyms", kerninfo->root_dir);
+                guest_kallsyms = path;
+        }
+
+        /*
+         * We use _stext for guest kernel because guest kernel's /proc/kallsyms
+         * have no _text sometimes.
+         */
+        err = event__synthesize_kernel_mmap(process_synthesized_event,
+                        session, kerninfo, "_text");
+        if (err < 0)
+                err = event__synthesize_kernel_mmap(process_synthesized_event,
+                                session, kerninfo, "_stext");
+        if (err < 0)
+                pr_err("Couldn't record guest kernel [%d]'s reference"
+                        " relocation symbol.\n", kerninfo->pid);
+}
+
 static int __cmd_record(int argc, const char **argv)
 {
         int i, counter;
@@ -467,6 +513,7 @@ static int __cmd_record(int argc, const char **argv)
         int child_ready_pipe[2], go_pipe[2];
         const bool forks = argc > 0;
         char buf;
+        struct kernel_info *kerninfo;
 
         page_size = sysconf(_SC_PAGE_SIZE);
 
@@ -635,21 +682,31 @@ static int __cmd_record(int argc, const char **argv)
                 advance_output(err);
         }
 
+        kerninfo = kerninfo__findhost(&session->kerninfo_root);
+        if (!kerninfo) {
+                pr_err("Couldn't find native kernel information.\n");
+                return -1;
+        }
+
         err = event__synthesize_kernel_mmap(process_synthesized_event,
-                                            session, "_text");
+                        session, kerninfo, "_text");
         if (err < 0)
                 err = event__synthesize_kernel_mmap(process_synthesized_event,
-                                                    session, "_stext");
+                                session, kerninfo, "_stext");
         if (err < 0) {
                 pr_err("Couldn't record kernel reference relocation symbol.\n");
                 return err;
         }
 
-        err = event__synthesize_modules(process_synthesized_event, session);
+        err = event__synthesize_modules(process_synthesized_event,
+                                session, kerninfo);
         if (err < 0) {
                 pr_err("Couldn't record kernel reference relocation symbol.\n");
                 return err;
         }
+        if (perf_guest)
+                kerninfo__process_allkernels(&session->kerninfo_root,
+                        event__synthesize_guest_os, session);
 
         if (!system_wide && profile_cpu == -1)
                 event__synthesize_thread(target_tid, process_synthesized_event,
diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c
index 7da5fb365264..816edae7c5b2 100644
--- a/tools/perf/builtin-report.c
+++ b/tools/perf/builtin-report.c
@@ -108,7 +108,7 @@ static int perf_session__add_hist_entry(struct perf_session *self,
                 return -ENOMEM;
 
         if (hit)
-                he->count += data->period;
+                __perf_session__add_count(he, al,  data->period);
 
         if (symbol_conf.use_callchain) {
                 if (!hit)
@@ -313,7 +313,7 @@ static int __cmd_report(void)
                 perf_session__fprintf(session, stdout);
 
         if (verbose > 2)
-                dsos__fprintf(stdout);
+                dsos__fprintf(&session->kerninfo_root, stdout);
 
         next = rb_first(&session->stats_by_id);
         while (next) {
@@ -450,6 +450,8 @@ static const struct option options[] = {
                    "sort by key(s): pid, comm, dso, symbol, parent"),
         OPT_BOOLEAN('P', "full-paths", &symbol_conf.full_paths,
                     "Don't shorten the pathnames taking into account the cwd"),
+        OPT_BOOLEAN(0, "showcpuutilization", &symbol_conf.show_cpu_utilization,
+                    "Show sample percentage for different cpu modes"),
         OPT_STRING('p', "parent", &parent_pattern, "regex",
                    "regex filter to identify parent, see: '--sort parent'"),
         OPT_BOOLEAN('x', "exclude-other", &symbol_conf.exclude_other,
diff --git a/tools/perf/builtin-top.c b/tools/perf/builtin-top.c
index 40f24dd46ef4..dfd7ea7dabdd 100644
--- a/tools/perf/builtin-top.c
+++ b/tools/perf/builtin-top.c
@@ -420,8 +420,9 @@ static double sym_weight(const struct sym_entry *sym)
 }
 
 static long                     samples;
-static long                     userspace_samples;
+static long                     kernel_samples, us_samples;
 static long                     exact_samples;
+static long                     guest_us_samples, guest_kernel_samples;
 static const char               CONSOLE_CLEAR[] = "";
 
 static void __list_insert_active_sym(struct sym_entry *syme)
@@ -461,7 +462,10 @@ static void print_sym_table(void)
         int printed = 0, j;
         int counter, snap = !display_weighted ? sym_counter : 0;
         float samples_per_sec = samples/delay_secs;
-        float ksamples_per_sec = (samples-userspace_samples)/delay_secs;
+        float ksamples_per_sec = kernel_samples/delay_secs;
+        float us_samples_per_sec = (us_samples)/delay_secs;
+        float guest_kernel_samples_per_sec = (guest_kernel_samples)/delay_secs;
+        float guest_us_samples_per_sec = (guest_us_samples)/delay_secs;
         float esamples_percent = (100.0*exact_samples)/samples;
         float sum_ksamples = 0.0;
         struct sym_entry *syme, *n;
@@ -470,7 +474,8 @@ static void print_sym_table(void)
         int sym_width = 0, dso_width = 0, dso_short_width = 0;
         const int win_width = winsize.ws_col - 1;
 
-        samples = userspace_samples = exact_samples = 0;
+        samples = us_samples = kernel_samples = exact_samples = 0;
+        guest_kernel_samples = guest_us_samples = 0;
 
         /* Sort the active symbols */
         pthread_mutex_lock(&active_symbols_lock);
@@ -501,10 +506,30 @@ static void print_sym_table(void)
         puts(CONSOLE_CLEAR);
 
         printf("%-*.*s\n", win_width, win_width, graph_dotted_line);
-        printf( "   PerfTop:%8.0f irqs/sec  kernel:%4.1f%%  exact: %4.1f%% [",
-                samples_per_sec,
-                100.0 - (100.0*((samples_per_sec-ksamples_per_sec)/samples_per_sec)),
-                esamples_percent);
+        if (!perf_guest) {
+                printf("   PerfTop:%8.0f irqs/sec  kernel:%4.1f%%"
+                        "  exact: %4.1f%% [",
+                        samples_per_sec,
+                        100.0 - (100.0 * ((samples_per_sec - ksamples_per_sec) /
+                                         samples_per_sec)),
+                        esamples_percent);
+        } else {
+                printf("   PerfTop:%8.0f irqs/sec  kernel:%4.1f%% us:%4.1f%%"
+                        " guest kernel:%4.1f%% guest us:%4.1f%%"
+                        " exact: %4.1f%% [",
+                        samples_per_sec,
+                        100.0 - (100.0 * ((samples_per_sec-ksamples_per_sec) /
+                                          samples_per_sec)),
+                        100.0 - (100.0 * ((samples_per_sec-us_samples_per_sec) /
+                                          samples_per_sec)),
+                        100.0 - (100.0 * ((samples_per_sec -
+                                                guest_kernel_samples_per_sec) /
+                                          samples_per_sec)),
+                        100.0 - (100.0 * ((samples_per_sec -
+                                           guest_us_samples_per_sec) /
+                                          samples_per_sec)),
+                        esamples_percent);
+        }
 
         if (nr_counters == 1 || !display_weighted) {
                 printf("%Ld", (u64)attrs[0].sample_period);
@@ -597,7 +622,6 @@ static void print_sym_table(void)
 
                 syme = rb_entry(nd, struct sym_entry, rb_node);
                 sym = sym_entry__symbol(syme);
-
                 if (++printed > print_entries || (int)syme->snap_count < count_filter)
                         continue;
 
@@ -761,7 +785,7 @@ static int key_mapped(int c)
         return 0;
 }
 
-static void handle_keypress(int c)
+static void handle_keypress(struct perf_session *session, int c)
 {
         if (!key_mapped(c)) {
                 struct pollfd stdin_poll = { .fd = 0, .events = POLLIN };
@@ -830,7 +854,7 @@ static void handle_keypress(int c)
                 case 'Q':
                         printf("exiting.\n");
                         if (dump_symtab)
-                                dsos__fprintf(stderr);
+                                dsos__fprintf(&session->kerninfo_root, stderr);
                         exit(0);
                 case 's':
                         prompt_symbol(&sym_filter_entry, "Enter details symbol");
@@ -866,6 +890,7 @@ static void *display_thread(void *arg __used)
         struct pollfd stdin_poll = { .fd = 0, .events = POLLIN };
         struct termios tc, save;
         int delay_msecs, c;
+        struct perf_session *session = (struct perf_session *) arg;
 
         tcgetattr(0, &save);
         tc = save;
@@ -886,7 +911,7 @@ repeat:
         c = getc(stdin);
         tcsetattr(0, TCSAFLUSH, &save);
 
-        handle_keypress(c);
+        handle_keypress(session, c);
         goto repeat;
 
         return NULL;
@@ -957,24 +982,46 @@ static void event__process_sample(const event_t *self,
         u64 ip = self->ip.ip;
         struct sym_entry *syme;
         struct addr_location al;
+        struct kernel_info *kerninfo;
         u8 origin = self->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
 
         ++samples;
 
         switch (origin) {
         case PERF_RECORD_MISC_USER:
-                ++userspace_samples;
+                ++us_samples;
                 if (hide_user_symbols)
                         return;
+                kerninfo = kerninfo__findhost(&session->kerninfo_root);
                 break;
         case PERF_RECORD_MISC_KERNEL:
+                ++kernel_samples;
                 if (hide_kernel_symbols)
                         return;
+                kerninfo = kerninfo__findhost(&session->kerninfo_root);
                 break;
+        case PERF_RECORD_MISC_GUEST_KERNEL:
+                ++guest_kernel_samples;
+                kerninfo = kerninfo__find(&session->kerninfo_root,
+                                          self->ip.pid);
+                break;
+        case PERF_RECORD_MISC_GUEST_USER:
+                ++guest_us_samples;
+                /*
+                 * TODO: we don't process guest user from host side
+                 * except simple counting.
+                 */
+                return;
         default:
                 return;
         }
 
+        if (!kerninfo && perf_guest) {
+                pr_err("Can't find guest [%d]'s kernel information\n",
+                        self->ip.pid);
+                return;
+        }
+
         if (self->header.misc & PERF_RECORD_MISC_EXACT)
                 exact_samples++;
 
@@ -994,7 +1041,7 @@ static void event__process_sample(const event_t *self,
                  * --hide-kernel-symbols, even if the user specifies an
                  * invalid --vmlinux ;-)
                  */
-                if (al.map == session->vmlinux_maps[MAP__FUNCTION] &&
+                if (al.map == kerninfo->vmlinux_maps[MAP__FUNCTION] &&
                     RB_EMPTY_ROOT(&al.map->dso->symbols[MAP__FUNCTION])) {
                         pr_err("The %s file can't be used\n",
                                symbol_conf.vmlinux_name);
@@ -1261,7 +1308,7 @@ static int __cmd_top(void)
 
         perf_session__mmap_read(session);
 
-        if (pthread_create(&thread, NULL, display_thread, NULL)) {
+        if (pthread_create(&thread, NULL, display_thread, session)) {
                 printf("Could not create display thread.\n");
                 exit(-1);
         }
diff --git a/tools/perf/builtin.h b/tools/perf/builtin.h
index 10fe49e7048a..ab28bca92e52 100644
--- a/tools/perf/builtin.h
+++ b/tools/perf/builtin.h
@@ -32,5 +32,6 @@ extern int cmd_version(int argc, const char **argv, const char *prefix);
 extern int cmd_probe(int argc, const char **argv, const char *prefix);
 extern int cmd_kmem(int argc, const char **argv, const char *prefix);
 extern int cmd_lock(int argc, const char **argv, const char *prefix);
+extern int cmd_kvm(int argc, const char **argv, const char *prefix);
 
 #endif
diff --git a/tools/perf/command-list.txt b/tools/perf/command-list.txt
index db6ee94d4a8e..2a1162d413a8 100644
--- a/tools/perf/command-list.txt
+++ b/tools/perf/command-list.txt
@@ -19,3 +19,4 @@ perf-trace			mainporcelain common
 perf-probe                      mainporcelain common
 perf-kmem                       mainporcelain common
 perf-lock                       mainporcelain common
+perf-kvm                        mainporcelain common
diff --git a/tools/perf/perf.c b/tools/perf/perf.c
index d4be55b6cd34..985cdb4bd005 100644
--- a/tools/perf/perf.c
+++ b/tools/perf/perf.c
@@ -307,6 +307,7 @@ static void handle_internal_command(int argc, const char **argv)
                 { "probe",      cmd_probe,      0 },
                 { "kmem",       cmd_kmem,       0 },
                 { "lock",       cmd_lock,       0 },
+                { "kvm",        cmd_kvm,        0 },
         };
         unsigned int i;
         static const char ext[] = STRIP_EXTENSION;
diff --git a/tools/perf/perf.h b/tools/perf/perf.h
index ec212748d651..02821febb704 100644
--- a/tools/perf/perf.h
+++ b/tools/perf/perf.h
@@ -131,4 +131,6 @@ struct ip_callchain {
         u64 ips[0];
 };
 
+extern int perf_host, perf_guest;
+
 #endif
diff --git a/tools/perf/util/build-id.c b/tools/perf/util/build-id.c
index 04904b35ba81..0f60a3906808 100644
--- a/tools/perf/util/build-id.c
+++ b/tools/perf/util/build-id.c
@@ -24,7 +24,7 @@ static int build_id__mark_dso_hit(event_t *event, struct perf_session *session)
         }
 
         thread__find_addr_map(thread, session, cpumode, MAP__FUNCTION,
-                              event->ip.ip, &al);
+                              event->ip.pid, event->ip.ip, &al);
 
         if (al.map != NULL)
                 al.map->dso->hit = 1;
diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c
index 571fb25f7eb9..e3fa8d3d11b4 100644
--- a/tools/perf/util/event.c
+++ b/tools/perf/util/event.c
@@ -112,7 +112,11 @@ static int event__synthesize_mmap_events(pid_t pid, pid_t tgid,
                 event_t ev = {
                         .header = {
                                 .type = PERF_RECORD_MMAP,
-                                .misc = 0, /* Just like the kernel, see kernel/perf_event.c __perf_event_mmap */
+                                /*
+                                 * Just like the kernel, see __perf_event_mmap
+                                 * in kernel/perf_event.c
+                                 */
+                                .misc = PERF_RECORD_MISC_USER,
                          },
                 };
                 int n;
@@ -167,11 +171,23 @@ static int event__synthesize_mmap_events(pid_t pid, pid_t tgid,
 }
 
 int event__synthesize_modules(event__handler_t process,
-                              struct perf_session *session)
+                              struct perf_session *session,
+                              struct kernel_info *kerninfo)
 {
         struct rb_node *nd;
+        struct map_groups *kmaps = &kerninfo->kmaps;
+        u16 misc;
 
-        for (nd = rb_first(&session->kmaps.maps[MAP__FUNCTION]);
+        /*
+         * kernel uses 0 for user space maps, see kernel/perf_event.c
+         * __perf_event_mmap
+         */
+        if (is_host_kernel(kerninfo))
+                misc = PERF_RECORD_MISC_KERNEL;
+        else
+                misc = PERF_RECORD_MISC_GUEST_KERNEL;
+
+        for (nd = rb_first(&kmaps->maps[MAP__FUNCTION]);
              nd; nd = rb_next(nd)) {
                 event_t ev;
                 size_t size;
@@ -182,12 +198,13 @@ int event__synthesize_modules(event__handler_t process,
 
                 size = ALIGN(pos->dso->long_name_len + 1, sizeof(u64));
                 memset(&ev, 0, sizeof(ev));
-                ev.mmap.header.misc = 1; /* kernel uses 0 for user space maps, see kernel/perf_event.c __perf_event_mmap */
+                ev.mmap.header.misc = misc;
                 ev.mmap.header.type = PERF_RECORD_MMAP;
                 ev.mmap.header.size = (sizeof(ev.mmap) -
                                         (sizeof(ev.mmap.filename) - size));
                 ev.mmap.start = pos->start;
                 ev.mmap.len   = pos->end - pos->start;
+                ev.mmap.pid   = kerninfo->pid;
 
                 memcpy(ev.mmap.filename, pos->dso->long_name,
                        pos->dso->long_name_len + 1);
@@ -250,13 +267,18 @@ static int find_symbol_cb(void *arg, const char *name, char type, u64 start)
 
 int event__synthesize_kernel_mmap(event__handler_t process,
                                   struct perf_session *session,
+                                  struct kernel_info *kerninfo,
                                   const char *symbol_name)
 {
         size_t size;
+        const char *filename, *mmap_name;
+        char path[PATH_MAX];
+        char name_buff[PATH_MAX];
+        struct map *map;
+
         event_t ev = {
                 .header = {
                         .type = PERF_RECORD_MMAP,
-                        .misc = 1, /* kernel uses 0 for user space maps, see kernel/perf_event.c __perf_event_mmap */
                 },
         };
         /*
@@ -266,16 +288,37 @@ int event__synthesize_kernel_mmap(event__handler_t process,
          */
         struct process_symbol_args args = { .name = symbol_name, };
 
-        if (kallsyms__parse("/proc/kallsyms", &args, find_symbol_cb) <= 0)
+        mmap_name = kern_mmap_name(kerninfo, name_buff);
+        if (is_host_kernel(kerninfo)) {
+                /*
+                 * kernel uses PERF_RECORD_MISC_USER for user space maps,
+                 * see kernel/perf_event.c __perf_event_mmap
+                 */
+                ev.header.misc = PERF_RECORD_MISC_KERNEL;
+                filename = "/proc/kallsyms";
+        } else {
+                ev.header.misc = PERF_RECORD_MISC_GUEST_KERNEL;
+                if (is_default_guest(kerninfo))
+                        filename = (char *) symbol_conf.default_guest_kallsyms;
+                else {
+                        sprintf(path, "%s/proc/kallsyms", kerninfo->root_dir);
+                        filename = path;
+                }
+        }
+
+        if (kallsyms__parse(filename, &args, find_symbol_cb) <= 0)
                 return -ENOENT;
 
+        map = kerninfo->vmlinux_maps[MAP__FUNCTION];
         size = snprintf(ev.mmap.filename, sizeof(ev.mmap.filename),
-                        "[kernel.kallsyms.%s]", symbol_name) + 1;
+                        "%s%s", mmap_name, symbol_name) + 1;
         size = ALIGN(size, sizeof(u64));
-        ev.mmap.header.size = (sizeof(ev.mmap) - (sizeof(ev.mmap.filename) - size));
+        ev.mmap.header.size = (sizeof(ev.mmap) -
+                        (sizeof(ev.mmap.filename) - size));
         ev.mmap.pgoff = args.start;
-        ev.mmap.start = session->vmlinux_maps[MAP__FUNCTION]->start;
-        ev.mmap.len   = session->vmlinux_maps[MAP__FUNCTION]->end - ev.mmap.start ;
+        ev.mmap.start = map->start;
+        ev.mmap.len   = map->end - ev.mmap.start;
+        ev.mmap.pid   = kerninfo->pid;
 
         return process(&ev, session);
 }
@@ -329,22 +372,50 @@ int event__process_lost(event_t *self, struct perf_session *session)
         return 0;
 }
 
-int event__process_mmap(event_t *self, struct perf_session *session)
+static void event_set_kernel_mmap_len(struct map **maps, event_t *self)
+{
+        maps[MAP__FUNCTION]->start = self->mmap.start;
+        maps[MAP__FUNCTION]->end   = self->mmap.start + self->mmap.len;
+        /*
+         * Be a bit paranoid here, some perf.data file came with
+         * a zero sized synthesized MMAP event for the kernel.
+         */
+        if (maps[MAP__FUNCTION]->end == 0)
+                maps[MAP__FUNCTION]->end = ~0UL;
+}
+
+static int event__process_kernel_mmap(event_t *self,
+                        struct perf_session *session)
 {
-        struct thread *thread;
         struct map *map;
+        char kmmap_prefix[PATH_MAX];
+        struct kernel_info *kerninfo;
+        enum dso_kernel_type kernel_type;
+        bool is_kernel_mmap;
+
+        kerninfo = kerninfo__findnew(&session->kerninfo_root, self->mmap.pid);
+        if (!kerninfo) {
+                pr_err("Can't find id %d's kerninfo\n", self->mmap.pid);
+                goto out_problem;
+        }
 
-        dump_printf(" %d/%d: [%#Lx(%#Lx) @ %#Lx]: %s\n",
-                    self->mmap.pid, self->mmap.tid, self->mmap.start,
-                    self->mmap.len, self->mmap.pgoff, self->mmap.filename);
+        kern_mmap_name(kerninfo, kmmap_prefix);
+        if (is_host_kernel(kerninfo))
+                kernel_type = DSO_TYPE_KERNEL;
+        else
+                kernel_type = DSO_TYPE_GUEST_KERNEL;
 
-        if (self->mmap.pid == 0) {
-                static const char kmmap_prefix[] = "[kernel.kallsyms.";
+        is_kernel_mmap = memcmp(self->mmap.filename,
+                                kmmap_prefix,
+                                strlen(kmmap_prefix)) == 0;
+        if (self->mmap.filename[0] == '/' ||
+            (!is_kernel_mmap && self->mmap.filename[0] == '[')) {
 
-                if (self->mmap.filename[0] == '/') {
-                        char short_module_name[1024];
-                        char *name = strrchr(self->mmap.filename, '/'), *dot;
+                char short_module_name[1024];
+                char *name, *dot;
 
+                if (self->mmap.filename[0] == '/') {
+                        name = strrchr(self->mmap.filename, '/');
                         if (name == NULL)
                                 goto out_problem;
 
@@ -352,59 +423,86 @@ int event__process_mmap(event_t *self, struct perf_session *session)
                         dot = strrchr(name, '.');
                         if (dot == NULL)
                                 goto out_problem;
-
                         snprintf(short_module_name, sizeof(short_module_name),
-                                 "[%.*s]", (int)(dot - name), name);
+                                        "[%.*s]", (int)(dot - name), name);
                         strxfrchar(short_module_name, '-', '_');
-
-                        map = perf_session__new_module_map(session,
-                                                           self->mmap.start,
-                                                           self->mmap.filename);
-                        if (map == NULL)
-                                goto out_problem;
-
-                        name = strdup(short_module_name);
-                        if (name == NULL)
-                                goto out_problem;
-
-                        map->dso->short_name = name;
-                        map->end = map->start + self->mmap.len;
-                } else if (memcmp(self->mmap.filename, kmmap_prefix,
-                                sizeof(kmmap_prefix) - 1) == 0) {
-                        const char *symbol_name = (self->mmap.filename +
-                                                   sizeof(kmmap_prefix) - 1);
+                } else
+                        strcpy(short_module_name, self->mmap.filename);
+
+                map = map_groups__new_module(&kerninfo->kmaps,
+                                self->mmap.start,
+                                self->mmap.filename,
+                                kerninfo);
+                if (map == NULL)
+                        goto out_problem;
+
+                name = strdup(short_module_name);
+                if (name == NULL)
+                        goto out_problem;
+
+                map->dso->short_name = name;
+                map->end = map->start + self->mmap.len;
+        } else if (is_kernel_mmap) {
+                const char *symbol_name = (self->mmap.filename +
+                                strlen(kmmap_prefix));
+                /*
+                 * Should be there already, from the build-id table in
+                 * the header.
+                 */
+                struct dso *kernel = __dsos__findnew(&kerninfo->dsos__kernel,
+                                kmmap_prefix);
+                if (kernel == NULL)
+                        goto out_problem;
+
+                kernel->kernel = kernel_type;
+                if (__map_groups__create_kernel_maps(&kerninfo->kmaps,
+                                        kerninfo->vmlinux_maps, kernel) < 0)
+                        goto out_problem;
+
+                event_set_kernel_mmap_len(kerninfo->vmlinux_maps, self);
+                perf_session__set_kallsyms_ref_reloc_sym(kerninfo->vmlinux_maps,
+                                symbol_name,
+                                self->mmap.pgoff);
+                if (is_default_guest(kerninfo)) {
                         /*
-                         * Should be there already, from the build-id table in
-                         * the header.
+                         * preload dso of guest kernel and modules
                          */
-                        struct dso *kernel = __dsos__findnew(&dsos__kernel,
-                                                             "[kernel.kallsyms]");
-                        if (kernel == NULL)
-                                goto out_problem;
-
-                        kernel->kernel = 1;
-                        if (__perf_session__create_kernel_maps(session, kernel) < 0)
-                                goto out_problem;
+                        dso__load(kernel,
+                                kerninfo->vmlinux_maps[MAP__FUNCTION],
+                                NULL);
+                }
+        }
+        return 0;
+out_problem:
+        return -1;
+}
 
-                        session->vmlinux_maps[MAP__FUNCTION]->start = self->mmap.start;
-                        session->vmlinux_maps[MAP__FUNCTION]->end   = self->mmap.start + self->mmap.len;
-                        /*
-                         * Be a bit paranoid here, some perf.data file came with
-                         * a zero sized synthesized MMAP event for the kernel.
-                         */
-                        if (session->vmlinux_maps[MAP__FUNCTION]->end == 0)
-                                session->vmlinux_maps[MAP__FUNCTION]->end = ~0UL;
+int event__process_mmap(event_t *self, struct perf_session *session)
+{
+        struct kernel_info *kerninfo;
+        struct thread *thread;
+        struct map *map;
+        u8 cpumode = self->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
+        int ret = 0;
 
-                        perf_session__set_kallsyms_ref_reloc_sym(session, symbol_name,
-                                                                 self->mmap.pgoff);
-                }
+        dump_printf(" %d/%d: [%#Lx(%#Lx) @ %#Lx]: %s\n",
+                        self->mmap.pid, self->mmap.tid, self->mmap.start,
+                        self->mmap.len, self->mmap.pgoff, self->mmap.filename);
+
+        if (cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
+            cpumode == PERF_RECORD_MISC_KERNEL) {
+                ret = event__process_kernel_mmap(self, session);
+                if (ret < 0)
+                        goto out_problem;
                 return 0;
         }
 
         thread = perf_session__findnew(session, self->mmap.pid);
-        map = map__new(self->mmap.start, self->mmap.len, self->mmap.pgoff,
-                       self->mmap.pid, self->mmap.filename, MAP__FUNCTION,
-                       session->cwd, session->cwdlen);
+        kerninfo = kerninfo__findhost(&session->kerninfo_root);
+        map = map__new(&kerninfo->dsos__user, self->mmap.start,
+                        self->mmap.len, self->mmap.pgoff,
+                        self->mmap.pid, self->mmap.filename,
+                        MAP__FUNCTION, session->cwd, session->cwdlen);
 
         if (thread == NULL || map == NULL)
                 goto out_problem;
@@ -444,22 +542,52 @@ int event__process_task(event_t *self, struct perf_session *session)
 
 void thread__find_addr_map(struct thread *self,
                            struct perf_session *session, u8 cpumode,
-                           enum map_type type, u64 addr,
+                           enum map_type type, pid_t pid, u64 addr,
                            struct addr_location *al)
 {
         struct map_groups *mg = &self->mg;
+        struct kernel_info *kerninfo = NULL;
 
         al->thread = self;
         al->addr = addr;
+        al->cpumode = cpumode;
+        al->filtered = false;
 
-        if (cpumode == PERF_RECORD_MISC_KERNEL) {
+        if (cpumode == PERF_RECORD_MISC_KERNEL && perf_host) {
                 al->level = 'k';
-                mg = &session->kmaps;
-        } else if (cpumode == PERF_RECORD_MISC_USER)
+                kerninfo = kerninfo__findhost(&session->kerninfo_root);
+                mg = &kerninfo->kmaps;
+        } else if (cpumode == PERF_RECORD_MISC_USER && perf_host) {
                 al->level = '.';
-        else {
-                al->level = 'H';
+                kerninfo = kerninfo__findhost(&session->kerninfo_root);
+        } else if (cpumode == PERF_RECORD_MISC_GUEST_KERNEL && perf_guest) {
+                al->level = 'g';
+                kerninfo = kerninfo__find(&session->kerninfo_root, pid);
+                if (!kerninfo) {
+                        al->map = NULL;
+                        return;
+                }
+                mg = &kerninfo->kmaps;
+        } else {
+                /*
+                 * 'u' means guest os user space.
+                 * TODO: We don't support guest user space. Might support late.
+                 */
+                if (cpumode == PERF_RECORD_MISC_GUEST_USER && perf_guest)
+                        al->level = 'u';
+                else
+                        al->level = 'H';
                 al->map = NULL;
+
+                if ((cpumode == PERF_RECORD_MISC_GUEST_USER ||
+                        cpumode == PERF_RECORD_MISC_GUEST_KERNEL) &&
+                        !perf_guest)
+                        al->filtered = true;
+                if ((cpumode == PERF_RECORD_MISC_USER ||
+                        cpumode == PERF_RECORD_MISC_KERNEL) &&
+                        !perf_host)
+                        al->filtered = true;
+
                 return;
         }
 try_again:
@@ -474,8 +602,11 @@ try_again:
                  * "[vdso]" dso, but for now lets use the old trick of looking
                  * in the whole kernel symbol list.
                  */
-                if ((long long)al->addr < 0 && mg != &session->kmaps) {
-                        mg = &session->kmaps;
+                if ((long long)al->addr < 0 &&
+                        cpumode == PERF_RECORD_MISC_KERNEL &&
+                        kerninfo &&
+                        mg != &kerninfo->kmaps)  {
+                        mg = &kerninfo->kmaps;
                         goto try_again;
                 }
         } else
@@ -484,11 +615,11 @@ try_again:
 
 void thread__find_addr_location(struct thread *self,
                                 struct perf_session *session, u8 cpumode,
-                                enum map_type type, u64 addr,
+                                enum map_type type, pid_t pid, u64 addr,
                                 struct addr_location *al,
                                 symbol_filter_t filter)
 {
-        thread__find_addr_map(self, session, cpumode, type, addr, al);
+        thread__find_addr_map(self, session, cpumode, type, pid, addr, al);
         if (al->map != NULL)
                 al->sym = map__find_symbol(al->map, al->addr, filter);
         else
@@ -524,7 +655,7 @@ int event__preprocess_sample(const event_t *self, struct perf_session *session,
         dump_printf(" ... thread: %s:%d\n", thread->comm, thread->pid);
 
         thread__find_addr_map(thread, session, cpumode, MAP__FUNCTION,
-                              self->ip.ip, al);
+                              self->ip.pid, self->ip.ip, al);
         dump_printf(" ...... dso: %s\n",
                     al->map ? al->map->dso->long_name :
                         al->level == 'H' ? "[hypervisor]" : "<not found>");
@@ -554,7 +685,6 @@ int event__preprocess_sample(const event_t *self, struct perf_session *session,
             !strlist__has_entry(symbol_conf.sym_list, al->sym->name))
                 goto out_filtered;
 
-        al->filtered = false;
         return 0;
 
 out_filtered:
diff --git a/tools/perf/util/event.h b/tools/perf/util/event.h
index e5740ea140ab..4af2ed5d48ad 100644
--- a/tools/perf/util/event.h
+++ b/tools/perf/util/event.h
@@ -79,6 +79,7 @@ struct sample_data {
 
 struct build_id_event {
         struct perf_event_header header;
+        pid_t                    pid;
         u8                       build_id[ALIGN(BUILD_ID_SIZE, sizeof(u64))];
         char                     filename[];
 };
@@ -154,10 +155,13 @@ int event__synthesize_thread(pid_t pid, event__handler_t process,
 void event__synthesize_threads(event__handler_t process,
                                struct perf_session *session);
 int event__synthesize_kernel_mmap(event__handler_t process,
-                                  struct perf_session *session,
-                                  const char *symbol_name);
+                                struct perf_session *session,
+                                struct kernel_info *kerninfo,
+                                const char *symbol_name);
+
 int event__synthesize_modules(event__handler_t process,
-                              struct perf_session *session);
+                              struct perf_session *session,
+                              struct kernel_info *kerninfo);
 
 int event__process_comm(event_t *self, struct perf_session *session);
 int event__process_lost(event_t *self, struct perf_session *session);
diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c
index 628173ba689e..75d016768021 100644
--- a/tools/perf/util/header.c
+++ b/tools/perf/util/header.c
@@ -190,7 +190,8 @@ static int write_padded(int fd, const void *bf, size_t count,
                         continue;               \
                 else
 
-static int __dsos__write_buildid_table(struct list_head *head, u16 misc, int fd)
+static int __dsos__write_buildid_table(struct list_head *head, pid_t pid,
+                                u16 misc, int fd)
 {
         struct dso *pos;
 
@@ -205,6 +206,7 @@ static int __dsos__write_buildid_table(struct list_head *head, u16 misc, int fd)
                 len = ALIGN(len, NAME_ALIGN);
                 memset(&b, 0, sizeof(b));
                 memcpy(&b.build_id, pos->build_id, sizeof(pos->build_id));
+                b.pid = pid;
                 b.header.misc = misc;
                 b.header.size = sizeof(b) + len;
                 err = do_write(fd, &b, sizeof(b));
@@ -219,13 +221,33 @@ static int __dsos__write_buildid_table(struct list_head *head, u16 misc, int fd)
         return 0;
 }
 
-static int dsos__write_buildid_table(int fd)
+static int dsos__write_buildid_table(struct perf_header *header, int fd)
 {
-        int err = __dsos__write_buildid_table(&dsos__kernel,
-                                              PERF_RECORD_MISC_KERNEL, fd);
-        if (err == 0)
-                err = __dsos__write_buildid_table(&dsos__user,
-                                                  PERF_RECORD_MISC_USER, fd);
+        struct perf_session *session = container_of(header,
+                        struct perf_session, header);
+        struct rb_node *nd;
+        int err = 0;
+        u16 kmisc, umisc;
+
+        for (nd = rb_first(&session->kerninfo_root); nd; nd = rb_next(nd)) {
+                struct kernel_info *pos = rb_entry(nd, struct kernel_info,
+                                rb_node);
+                if (is_host_kernel(pos)) {
+                        kmisc = PERF_RECORD_MISC_KERNEL;
+                        umisc = PERF_RECORD_MISC_USER;
+                } else {
+                        kmisc = PERF_RECORD_MISC_GUEST_KERNEL;
+                        umisc = PERF_RECORD_MISC_GUEST_USER;
+                }
+
+                err = __dsos__write_buildid_table(&pos->dsos__kernel, pos->pid,
+                                kmisc, fd);
+                if (err == 0)
+                        err = __dsos__write_buildid_table(&pos->dsos__user,
+                                pos->pid, umisc, fd);
+                if (err)
+                        break;
+        }
         return err;
 }
 
@@ -342,9 +364,12 @@ static int __dsos__cache_build_ids(struct list_head *head, const char *debugdir)
         return err;
 }
 
-static int dsos__cache_build_ids(void)
+static int dsos__cache_build_ids(struct perf_header *self)
 {
-        int err_kernel, err_user;
+        struct perf_session *session = container_of(self,
+                        struct perf_session, header);
+        struct rb_node *nd;
+        int ret = 0;
         char debugdir[PATH_MAX];
 
         snprintf(debugdir, sizeof(debugdir), "%s/%s", getenv("HOME"),
@@ -353,9 +378,30 @@ static int dsos__cache_build_ids(void)
         if (mkdir(debugdir, 0755) != 0 && errno != EEXIST)
                 return -1;
 
-        err_kernel = __dsos__cache_build_ids(&dsos__kernel, debugdir);
-        err_user   = __dsos__cache_build_ids(&dsos__user, debugdir);
-        return err_kernel || err_user ? -1 : 0;
+        for (nd = rb_first(&session->kerninfo_root); nd; nd = rb_next(nd)) {
+                struct kernel_info *pos = rb_entry(nd, struct kernel_info,
+                                rb_node);
+                ret |= __dsos__cache_build_ids(&pos->dsos__kernel, debugdir);
+                ret |= __dsos__cache_build_ids(&pos->dsos__user, debugdir);
+        }
+        return ret ? -1 : 0;
+}
+
+static bool dsos__read_build_ids(struct perf_header *self, bool with_hits)
+{
+        bool ret = false;
+        struct perf_session *session = container_of(self,
+                        struct perf_session, header);
+        struct rb_node *nd;
+
+        for (nd = rb_first(&session->kerninfo_root); nd; nd = rb_next(nd)) {
+                struct kernel_info *pos = rb_entry(nd, struct kernel_info,
+                                rb_node);
+                ret |= __dsos__read_build_ids(&pos->dsos__kernel, with_hits);
+                ret |= __dsos__read_build_ids(&pos->dsos__user, with_hits);
+        }
+
+        return ret;
 }
 
 static int perf_header__adds_write(struct perf_header *self, int fd)
@@ -366,7 +412,7 @@ static int perf_header__adds_write(struct perf_header *self, int fd)
         u64 sec_start;
         int idx = 0, err;
 
-        if (dsos__read_build_ids(true))
+        if (dsos__read_build_ids(self, true))
                 perf_header__set_feat(self, HEADER_BUILD_ID);
 
         nr_sections = bitmap_weight(self->adds_features, HEADER_FEAT_BITS);
@@ -401,14 +447,14 @@ static int perf_header__adds_write(struct perf_header *self, int fd)
 
                 /* Write build-ids */
                 buildid_sec->offset = lseek(fd, 0, SEEK_CUR);
-                err = dsos__write_buildid_table(fd);
+                err = dsos__write_buildid_table(self, fd);
                 if (err < 0) {
                         pr_debug("failed to write buildid table\n");
                         goto out_free;
                 }
                 buildid_sec->size = lseek(fd, 0, SEEK_CUR) -
                                           buildid_sec->offset;
-                dsos__cache_build_ids();
+                dsos__cache_build_ids(self);
         }
 
         lseek(fd, sec_start, SEEK_SET);
@@ -633,6 +679,85 @@ int perf_file_header__read(struct perf_file_header *self,
         return 0;
 }
 
+static int __event_process_build_id(struct build_id_event *bev,
+                                    char *filename,
+                                    struct perf_session *session)
+{
+        int err = -1;
+        struct list_head *head;
+        struct kernel_info *kerninfo;
+        u16 misc;
+        struct dso *dso;
+        enum dso_kernel_type dso_type;
+
+        kerninfo = kerninfo__findnew(&session->kerninfo_root, bev->pid);
+        if (!kerninfo)
+                goto out;
+
+        misc = bev->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
+
+        switch (misc) {
+        case PERF_RECORD_MISC_KERNEL:
+                dso_type = DSO_TYPE_KERNEL;
+                head = &kerninfo->dsos__kernel;
+                break;
+        case PERF_RECORD_MISC_GUEST_KERNEL:
+                dso_type = DSO_TYPE_GUEST_KERNEL;
+                head = &kerninfo->dsos__kernel;
+                break;
+        case PERF_RECORD_MISC_USER:
+        case PERF_RECORD_MISC_GUEST_USER:
+                dso_type = DSO_TYPE_USER;
+                head = &kerninfo->dsos__user;
+                break;
+        default:
+                goto out;
+        }
+
+        dso = __dsos__findnew(head, filename);
+        if (dso != NULL) {
+                dso__set_build_id(dso, &bev->build_id);
+                        if (filename[0] == '[')
+                                dso->kernel = dso_type;
+                }
+
+        err = 0;
+out:
+        return err;
+}
+
+static int perf_header__read_build_ids(struct perf_header *self,
+                        int input, u64 offset, u64 size)
+{
+        struct perf_session *session = container_of(self,
+                        struct perf_session, header);
+        struct build_id_event bev;
+        char filename[PATH_MAX];
+        u64 limit = offset + size;
+        int err = -1;
+
+        while (offset < limit) {
+                ssize_t len;
+
+                if (read(input, &bev, sizeof(bev)) != sizeof(bev))
+                        goto out;
+
+                if (self->needs_swap)
+                        perf_event_header__bswap(&bev.header);
+
+                len = bev.header.size - sizeof(bev);
+                if (read(input, filename, len) != len)
+                        goto out;
+
+                __event_process_build_id(&bev, filename, session);
+
+                offset += bev.header.size;
+        }
+        err = 0;
+out:
+        return err;
+}
+
 static int perf_file_section__process(struct perf_file_section *self,
                                       struct perf_header *ph,
                                       int feat, int fd)
@@ -989,6 +1114,7 @@ int event__process_tracing_data(event_t *self,
 
 int event__synthesize_build_id(struct dso *pos, u16 misc,
                                event__handler_t process,
+                               struct kernel_info *kerninfo,
                                struct perf_session *session)
 {
         event_t ev;
@@ -1005,6 +1131,7 @@ int event__synthesize_build_id(struct dso *pos, u16 misc,
         memcpy(&ev.build_id.build_id, pos->build_id, sizeof(pos->build_id));
         ev.build_id.header.type = PERF_RECORD_HEADER_BUILD_ID;
         ev.build_id.header.misc = misc;
+        ev.build_id.pid = kerninfo->pid;
         ev.build_id.header.size = sizeof(ev.build_id) + len;
         memcpy(&ev.build_id.filename, pos->long_name, pos->long_name_len);
 
@@ -1015,6 +1142,7 @@ int event__synthesize_build_id(struct dso *pos, u16 misc,
 
 static int __event_synthesize_build_ids(struct list_head *head, u16 misc,
                                         event__handler_t process,
+                                        struct kernel_info *kerninfo,
                                         struct perf_session *session)
 {
         struct dso *pos;
@@ -1024,7 +1152,8 @@ static int __event_synthesize_build_ids(struct list_head *head, u16 misc,
                 if (!pos->hit)
                         continue;
 
-                err = event__synthesize_build_id(pos, misc, process, session);
+                err = event__synthesize_build_id(pos, misc, process,
+                                        kerninfo, session);
                 if (err < 0)
                         return err;
         }
@@ -1035,44 +1164,48 @@ static int __event_synthesize_build_ids(struct list_head *head, u16 misc,
 int event__synthesize_build_ids(event__handler_t process,
                                 struct perf_session *session)
 {
-        int err;
+        int err = 0;
+        u16 kmisc, umisc;
+        struct kernel_info *pos;
+        struct rb_node *nd;
 
-        if (!dsos__read_build_ids(true))
+        if (!dsos__read_build_ids(&session->header, true))
                 return 0;
 
-        err = __event_synthesize_build_ids(&dsos__kernel,
-                                           PERF_RECORD_MISC_KERNEL,
-                                           process, session);
-        if (err == 0)
-                err = __event_synthesize_build_ids(&dsos__user,
-                                                   PERF_RECORD_MISC_USER,
-                                                   process, session);
+        for (nd = rb_first(&session->kerninfo_root); nd; nd = rb_next(nd)) {
+                pos = rb_entry(nd, struct kernel_info, rb_node);
+                if (is_host_kernel(pos)) {
+                        kmisc = PERF_RECORD_MISC_KERNEL;
+                        umisc = PERF_RECORD_MISC_USER;
+                } else {
+                        kmisc = PERF_RECORD_MISC_GUEST_KERNEL;
+                        umisc = PERF_RECORD_MISC_GUEST_USER;
+                }
+
+                err = __event_synthesize_build_ids(&pos->dsos__kernel,
+                                kmisc, process, pos, session);
+                if (err == 0)
+                        err = __event_synthesize_build_ids(&pos->dsos__user,
+                                        umisc, process, pos, session);
+                if (err)
+                        break;
+        }
 
         if (err < 0) {
                 pr_debug("failed to synthesize build ids\n");
                 return err;
         }
 
-        dsos__cache_build_ids();
+        dsos__cache_build_ids(&session->header);
 
         return 0;
 }
 
 int event__process_build_id(event_t *self,
-                            struct perf_session *session __unused)
+                            struct perf_session *session)
 {
-        struct list_head *head = &dsos__user;
-        struct dso *dso;
-
-        if (self->build_id.header.misc & PERF_RECORD_MISC_KERNEL)
-                head = &dsos__kernel;
-
-        dso = __dsos__findnew(head, self->build_id.filename);
-        if (dso != NULL) {
-                dso__set_build_id(dso, &self->build_id.build_id);
-                if (head == &dsos__kernel && self->build_id.filename[0] == '[')
-                        dso->kernel = 1;
-        }
-
+        __event_process_build_id(&self->build_id,
+                                 self->build_id.filename,
+                                 session);
         return 0;
 }
diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h
index 4214e2375650..275915458148 100644
--- a/tools/perf/util/header.h
+++ b/tools/perf/util/header.h
@@ -120,6 +120,7 @@ int event__process_tracing_data(event_t *self,
 
 int event__synthesize_build_id(struct dso *pos, u16 misc,
                                event__handler_t process,
+                               struct kernel_info *kerninfo,
                                struct perf_session *session);
 int event__synthesize_build_ids(event__handler_t process,
                                 struct perf_session *session);
diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c
index 9c2b8743cef6..ad6b22dde27f 100644
--- a/tools/perf/util/hist.c
+++ b/tools/perf/util/hist.c
@@ -8,6 +8,30 @@ struct callchain_param	callchain_param = {
         .min_percent = 0.5
 };
 
+void __perf_session__add_count(struct hist_entry *he,
+                        struct addr_location *al,
+                        u64 count)
+{
+        he->count += count;
+
+        switch (al->cpumode) {
+        case PERF_RECORD_MISC_KERNEL:
+                he->count_sys += count;
+                break;
+        case PERF_RECORD_MISC_USER:
+                he->count_us += count;
+                break;
+        case PERF_RECORD_MISC_GUEST_KERNEL:
+                he->count_guest_sys += count;
+                break;
+        case PERF_RECORD_MISC_GUEST_USER:
+                he->count_guest_us += count;
+                break;
+        default:
+                break;
+        }
+}
+
 /*
  * histogram, sorted on item, collects counts
  */
@@ -464,7 +488,7 @@ int hist_entry__snprintf(struct hist_entry *self,
                            u64 session_total)
 {
         struct sort_entry *se;
-        u64 count, total;
+        u64 count, total, count_sys, count_us, count_guest_sys, count_guest_us;
         const char *sep = symbol_conf.field_sep;
         int ret;
 
@@ -474,9 +498,17 @@ int hist_entry__snprintf(struct hist_entry *self,
         if (pair_session) {
                 count = self->pair ? self->pair->count : 0;
                 total = pair_session->events_stats.total;
+                count_sys = self->pair ? self->pair->count_sys : 0;
+                count_us = self->pair ? self->pair->count_us : 0;
+                count_guest_sys = self->pair ? self->pair->count_guest_sys : 0;
+                count_guest_us = self->pair ? self->pair->count_guest_us : 0;
         } else {
                 count = self->count;
                 total = session_total;
+                count_sys = self->count_sys;
+                count_us = self->count_us;
+                count_guest_sys = self->count_guest_sys;
+                count_guest_us = self->count_guest_us;
         }
 
         if (total) {
@@ -487,6 +519,26 @@ int hist_entry__snprintf(struct hist_entry *self,
                 else
                         ret = snprintf(s, size, sep ? "%.2f" : "   %6.2f%%",
                                        (count * 100.0) / total);
+                if (symbol_conf.show_cpu_utilization) {
+                        ret += percent_color_snprintf(s + ret, size - ret,
+                                        sep ? "%.2f" : "   %6.2f%%",
+                                        (count_sys * 100.0) / total);
+                        ret += percent_color_snprintf(s + ret, size - ret,
+                                        sep ? "%.2f" : "   %6.2f%%",
+                                        (count_us * 100.0) / total);
+                        if (perf_guest) {
+                                ret += percent_color_snprintf(s + ret,
+                                                size - ret,
+                                                sep ? "%.2f" : "   %6.2f%%",
+                                                (count_guest_sys * 100.0) /
+                                                                total);
+                                ret += percent_color_snprintf(s + ret,
+                                                size - ret,
+                                                sep ? "%.2f" : "   %6.2f%%",
+                                                (count_guest_us * 100.0) /
+                                                                total);
+                        }
+                }
         } else
                 ret = snprintf(s, size, sep ? "%lld" : "%12lld ", count);
 
@@ -597,6 +649,24 @@ size_t perf_session__fprintf_hists(struct rb_root *hists,
                         fputs("  Samples  ", fp);
         }
 
+        if (symbol_conf.show_cpu_utilization) {
+                if (sep) {
+                        ret += fprintf(fp, "%csys", *sep);
+                        ret += fprintf(fp, "%cus", *sep);
+                        if (perf_guest) {
+                                ret += fprintf(fp, "%cguest sys", *sep);
+                                ret += fprintf(fp, "%cguest us", *sep);
+                        }
+                } else {
+                        ret += fprintf(fp, "  sys  ");
+                        ret += fprintf(fp, "  us  ");
+                        if (perf_guest) {
+                                ret += fprintf(fp, "  guest sys  ");
+                                ret += fprintf(fp, "  guest us  ");
+                        }
+                }
+        }
+
         if (pair) {
                 if (sep)
                         ret += fprintf(fp, "%cDelta", *sep);
diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h
index ad17f0ad798b..9df1c340ec92 100644
--- a/tools/perf/util/hist.h
+++ b/tools/perf/util/hist.h
@@ -12,6 +12,9 @@ struct addr_location;
 struct symbol;
 struct rb_root;
 
+void __perf_session__add_count(struct hist_entry *he,
+                        struct addr_location *al,
+                        u64 count);
 struct hist_entry *__perf_session__add_hist_entry(struct rb_root *hists,
                                                   struct addr_location *al,
                                                   struct symbol *parent,
diff --git a/tools/perf/util/map.c b/tools/perf/util/map.c
index 37913b241bdf..7facd016ec97 100644
--- a/tools/perf/util/map.c
+++ b/tools/perf/util/map.c
@@ -4,6 +4,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <stdio.h>
+#include <unistd.h>
 #include "map.h"
 
 const char *map_type__name[MAP__NR_TYPES] = {
@@ -37,9 +38,11 @@ void map__init(struct map *self, enum map_type type,
         self->map_ip   = map__map_ip;
         self->unmap_ip = map__unmap_ip;
         RB_CLEAR_NODE(&self->rb_node);
+        self->groups   = NULL;
 }
 
-struct map *map__new(u64 start, u64 len, u64 pgoff, u32 pid, char *filename,
+struct map *map__new(struct list_head *dsos__list, u64 start, u64 len,
+                     u64 pgoff, u32 pid, char *filename,
                      enum map_type type, char *cwd, int cwdlen)
 {
         struct map *self = malloc(sizeof(*self));
@@ -66,7 +69,7 @@ struct map *map__new(u64 start, u64 len, u64 pgoff, u32 pid, char *filename,
                         filename = newfilename;
                 }
 
-                dso = dsos__findnew(filename);
+                dso = __dsos__findnew(dsos__list, filename);
                 if (dso == NULL)
                         goto out_delete;
 
@@ -242,6 +245,7 @@ void map_groups__init(struct map_groups *self)
                 self->maps[i] = RB_ROOT;
                 INIT_LIST_HEAD(&self->removed_maps[i]);
         }
+         self->this_kerninfo = NULL;
 }
 
 void map_groups__flush(struct map_groups *self)
@@ -508,3 +512,134 @@ struct map *maps__find(struct rb_root *maps, u64 ip)
 
         return NULL;
 }
+
+struct kernel_info *add_new_kernel_info(struct rb_root *kerninfo_root,
+                        pid_t pid, const char *root_dir)
+{
+        struct rb_node **p = &kerninfo_root->rb_node;
+        struct rb_node *parent = NULL;
+        struct kernel_info *kerninfo, *pos;
+
+        kerninfo = malloc(sizeof(struct kernel_info));
+        if (!kerninfo)
+                return NULL;
+
+        kerninfo->pid = pid;
+        map_groups__init(&kerninfo->kmaps);
+        kerninfo->root_dir = strdup(root_dir);
+        RB_CLEAR_NODE(&kerninfo->rb_node);
+        INIT_LIST_HEAD(&kerninfo->dsos__user);
+        INIT_LIST_HEAD(&kerninfo->dsos__kernel);
+        kerninfo->kmaps.this_kerninfo = kerninfo;
+
+        while (*p != NULL) {
+                parent = *p;
+                pos = rb_entry(parent, struct kernel_info, rb_node);
+                if (pid < pos->pid)
+                        p = &(*p)->rb_left;
+                else
+                        p = &(*p)->rb_right;
+        }
+
+        rb_link_node(&kerninfo->rb_node, parent, p);
+        rb_insert_color(&kerninfo->rb_node, kerninfo_root);
+
+        return kerninfo;
+}
+
+struct kernel_info *kerninfo__find(struct rb_root *kerninfo_root, pid_t pid)
+{
+        struct rb_node **p = &kerninfo_root->rb_node;
+        struct rb_node *parent = NULL;
+        struct kernel_info *kerninfo;
+        struct kernel_info *default_kerninfo = NULL;
+
+        while (*p != NULL) {
+                parent = *p;
+                kerninfo = rb_entry(parent, struct kernel_info, rb_node);
+                if (pid < kerninfo->pid)
+                        p = &(*p)->rb_left;
+                else if (pid > kerninfo->pid)
+                        p = &(*p)->rb_right;
+                else
+                        return kerninfo;
+                if (!kerninfo->pid)
+                        default_kerninfo = kerninfo;
+        }
+
+        return default_kerninfo;
+}
+
+struct kernel_info *kerninfo__findhost(struct rb_root *kerninfo_root)
+{
+        struct rb_node **p = &kerninfo_root->rb_node;
+        struct rb_node *parent = NULL;
+        struct kernel_info *kerninfo;
+        pid_t pid = HOST_KERNEL_ID;
+
+        while (*p != NULL) {
+                parent = *p;
+                kerninfo = rb_entry(parent, struct kernel_info, rb_node);
+                if (pid < kerninfo->pid)
+                        p = &(*p)->rb_left;
+                else if (pid > kerninfo->pid)
+                        p = &(*p)->rb_right;
+                else
+                        return kerninfo;
+        }
+
+        return NULL;
+}
+
+struct kernel_info *kerninfo__findnew(struct rb_root *kerninfo_root, pid_t pid)
+{
+        char path[PATH_MAX];
+        const char *root_dir;
+        int ret;
+        struct kernel_info *kerninfo = kerninfo__find(kerninfo_root, pid);
+
+        if (!kerninfo || kerninfo->pid != pid) {
+                if (pid == HOST_KERNEL_ID || pid == DEFAULT_GUEST_KERNEL_ID)
+                        root_dir = "";
+                else {
+                        if (!symbol_conf.guestmount)
+                                goto out;
+                        sprintf(path, "%s/%d", symbol_conf.guestmount, pid);
+                        ret = access(path, R_OK);
+                        if (ret) {
+                                pr_err("Can't access file %s\n", path);
+                                goto out;
+                        }
+                        root_dir = path;
+                }
+                kerninfo = add_new_kernel_info(kerninfo_root, pid, root_dir);
+        }
+
+out:
+        return kerninfo;
+}
+
+void kerninfo__process_allkernels(struct rb_root *kerninfo_root,
+                process_kernel_info process,
+                void *data)
+{
+        struct rb_node *nd;
+
+        for (nd = rb_first(kerninfo_root); nd; nd = rb_next(nd)) {
+                struct kernel_info *pos = rb_entry(nd, struct kernel_info,
+                                                        rb_node);
+                process(pos, data);
+        }
+}
+
+char *kern_mmap_name(struct kernel_info *kerninfo, char *buff)
+{
+        if (is_host_kernel(kerninfo))
+                sprintf(buff, "[%s]", "kernel.kallsyms");
+        else if (is_default_guest(kerninfo))
+                sprintf(buff, "[%s]", "guest.kernel.kallsyms");
+        else
+                sprintf(buff, "[%s.%d]", "guest.kernel.kallsyms", kerninfo->pid);
+
+        return buff;
+}
diff --git a/tools/perf/util/map.h b/tools/perf/util/map.h
index 2031278cc06a..30d38d634e09 100644
--- a/tools/perf/util/map.h
+++ b/tools/perf/util/map.h
@@ -19,6 +19,7 @@ extern const char *map_type__name[MAP__NR_TYPES];
 struct dso;
 struct ref_reloc_sym;
 struct map_groups;
+struct kernel_info;
 
 struct map {
         union {
@@ -36,6 +37,7 @@ struct map {
         u64                     (*unmap_ip)(struct map *, u64);
 
         struct dso              *dso;
+        struct map_groups       *groups;
 };
 
 struct kmap {
@@ -43,6 +45,26 @@ struct kmap {
         struct map_groups       *kmaps;
 };
 
+struct map_groups {
+        struct rb_root          maps[MAP__NR_TYPES];
+        struct list_head        removed_maps[MAP__NR_TYPES];
+        struct kernel_info      *this_kerninfo;
+};
+
+/* Native host kernel uses -1 as pid index in kernel_info */
+#define HOST_KERNEL_ID                  (-1)
+#define DEFAULT_GUEST_KERNEL_ID         (0)
+
+struct kernel_info {
+        struct rb_node rb_node;
+        pid_t pid;
+        char *root_dir;
+        struct list_head dsos__user;
+        struct list_head dsos__kernel;
+        struct map_groups kmaps;
+        struct map *vmlinux_maps[MAP__NR_TYPES];
+};
+
 static inline struct kmap *map__kmap(struct map *self)
 {
         return (struct kmap *)(self + 1);
@@ -74,7 +96,8 @@ typedef int (*symbol_filter_t)(struct map *map, struct symbol *sym);
 
 void map__init(struct map *self, enum map_type type,
                u64 start, u64 end, u64 pgoff, struct dso *dso);
-struct map *map__new(u64 start, u64 len, u64 pgoff, u32 pid, char *filename,
+struct map *map__new(struct list_head *dsos__list, u64 start, u64 len,
+                     u64 pgoff, u32 pid, char *filename,
                      enum map_type type, char *cwd, int cwdlen);
 void map__delete(struct map *self);
 struct map *map__clone(struct map *self);
@@ -91,11 +114,6 @@ void map__fixup_end(struct map *self);
 
 void map__reloc_vmlinux(struct map *self);
 
-struct map_groups {
-        struct rb_root          maps[MAP__NR_TYPES];
-        struct list_head        removed_maps[MAP__NR_TYPES];
-};
-
 size_t __map_groups__fprintf_maps(struct map_groups *self,
                                   enum map_type type, int verbose, FILE *fp);
 void maps__insert(struct rb_root *maps, struct map *map);
@@ -106,9 +124,40 @@ int map_groups__clone(struct map_groups *self,
 size_t map_groups__fprintf(struct map_groups *self, int verbose, FILE *fp);
 size_t map_groups__fprintf_maps(struct map_groups *self, int verbose, FILE *fp);
 
+struct kernel_info *add_new_kernel_info(struct rb_root *kerninfo_root,
+                        pid_t pid, const char *root_dir);
+struct kernel_info *kerninfo__find(struct rb_root *kerninfo_root, pid_t pid);
+struct kernel_info *kerninfo__findnew(struct rb_root *kerninfo_root, pid_t pid);
+struct kernel_info *kerninfo__findhost(struct rb_root *kerninfo_root);
+char *kern_mmap_name(struct kernel_info *kerninfo, char *buff);
+
+/*
+ * Default guest kernel is defined by parameter --guestkallsyms
+ * and --guestmodules
+ */
+static inline int is_default_guest(struct kernel_info *kerninfo)
+{
+        if (!kerninfo)
+                return 0;
+        return kerninfo->pid == DEFAULT_GUEST_KERNEL_ID;
+}
+
+static inline int is_host_kernel(struct kernel_info *kerninfo)
+{
+        if (!kerninfo)
+                return 0;
+        return kerninfo->pid == HOST_KERNEL_ID;
+}
+
+typedef void (*process_kernel_info)(struct kernel_info *kerninfo, void *data);
+void kerninfo__process_allkernels(struct rb_root *kerninfo_root,
+                                  process_kernel_info process,
+                                  void *data);
+
 static inline void map_groups__insert(struct map_groups *self, struct map *map)
 {
-         maps__insert(&self->maps[map->type], map);
+        maps__insert(&self->maps[map->type], map);
+        map->groups = self;
 }
 
 static inline struct map *map_groups__find(struct map_groups *self,
@@ -148,13 +197,11 @@ int map_groups__fixup_overlappings(struct map_groups *self, struct map *map,
 
 struct map *map_groups__find_by_name(struct map_groups *self,
                                      enum map_type type, const char *name);
-int __map_groups__create_kernel_maps(struct map_groups *self,
-                                     struct map *vmlinux_maps[MAP__NR_TYPES],
-                                     struct dso *kernel);
-int map_groups__create_kernel_maps(struct map_groups *self,
-                                   struct map *vmlinux_maps[MAP__NR_TYPES]);
-struct map *map_groups__new_module(struct map_groups *self, u64 start,
-                                   const char *filename);
+struct map *map_groups__new_module(struct map_groups *self,
+                                    u64 start,
+                                    const char *filename,
+                                    struct kernel_info *kerninfo);
+
 void map_groups__flush(struct map_groups *self);
 
 #endif /* __PERF_MAP_H */
diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
index 5bf8ab034466..3967f8f63d0d 100644
--- a/tools/perf/util/probe-event.c
+++ b/tools/perf/util/probe-event.c
@@ -78,6 +78,7 @@ static struct map *kmaps[MAP__NR_TYPES];
 /* Initialize symbol maps and path of vmlinux */
 static int init_vmlinux(void)
 {
+        struct dso *kernel;
         int ret;
 
         symbol_conf.sort_by_name = true;
@@ -91,8 +92,12 @@ static int init_vmlinux(void)
                 goto out;
         }
 
+        kernel = dso__new_kernel(symbol_conf.vmlinux_name);
+        if (kernel == NULL)
+                die("Failed to create kernel dso.");
+
         map_groups__init(&kmap_groups);
-        ret = map_groups__create_kernel_maps(&kmap_groups, kmaps);
+        ret = __map_groups__create_kernel_maps(&kmap_groups, kmaps, kernel);
         if (ret < 0)
                 pr_debug("Failed to create kernel maps.\n");
 
diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c
index 0fdf3ebef1e9..7d88ae5c270f 100644
--- a/tools/perf/util/session.c
+++ b/tools/perf/util/session.c
@@ -67,6 +67,17 @@ void perf_session__update_sample_type(struct perf_session *self)
         self->sample_type = perf_header__sample_type(&self->header);
 }
 
+int perf_session__create_kernel_maps(struct perf_session *self)
+{
+        int ret;
+        struct rb_root *root = &self->kerninfo_root;
+
+        ret = map_groups__create_kernel_maps(root, HOST_KERNEL_ID);
+        if (ret >= 0)
+                ret = map_groups__create_guest_kernel_maps(root);
+        return ret;
+}
+
 struct perf_session *perf_session__new(const char *filename, int mode, bool force)
 {
         size_t len = filename ? strlen(filename) + 1 : 0;
@@ -86,7 +97,7 @@ struct perf_session *perf_session__new(const char *filename, int mode, bool forc
         self->cwd = NULL;
         self->cwdlen = 0;
         self->unknown_events = 0;
-        map_groups__init(&self->kmaps);
+        self->kerninfo_root = RB_ROOT;
 
         if (mode == O_RDONLY) {
                 if (perf_session__open(self, force) < 0)
@@ -157,8 +168,9 @@ struct map_symbol *perf_session__resolve_callchain(struct perf_session *self,
                         continue;
                 }
 
+                al.filtered = false;
                 thread__find_addr_location(thread, self, cpumode,
-                                           MAP__FUNCTION, ip, &al, NULL);
+                                MAP__FUNCTION, thread->pid, ip, &al, NULL);
                 if (al.sym != NULL) {
                         if (sort__has_parent && !*parent &&
                             symbol__match_parent_regex(al.sym))
@@ -399,46 +411,6 @@ void perf_event_header__bswap(struct perf_event_header *self)
         self->size = bswap_16(self->size);
 }
 
-int perf_header__read_build_ids(struct perf_header *self,
-                                int input, u64 offset, u64 size)
-{
-        struct build_id_event bev;
-        char filename[PATH_MAX];
-        u64 limit = offset + size;
-        int err = -1;
-
-        while (offset < limit) {
-                struct dso *dso;
-                ssize_t len;
-                struct list_head *head = &dsos__user;
-
-                if (read(input, &bev, sizeof(bev)) != sizeof(bev))
-                        goto out;
-
-                if (self->needs_swap)
-                        perf_event_header__bswap(&bev.header);
-
-                len = bev.header.size - sizeof(bev);
-                if (read(input, filename, len) != len)
-                        goto out;
-
-                if (bev.header.misc & PERF_RECORD_MISC_KERNEL)
-                        head = &dsos__kernel;
-
-                dso = __dsos__findnew(head, filename);
-                if (dso != NULL) {
-                        dso__set_build_id(dso, &bev.build_id);
-                        if (head == &dsos__kernel && filename[0] == '[')
-                                dso->kernel = 1;
-                }
-
-                offset += bev.header.size;
-        }
-        err = 0;
-out:
-        return err;
-}
-
 static struct thread *perf_session__register_idle_thread(struct perf_session *self)
 {
         struct thread *thread = perf_session__findnew(self, 0);
@@ -690,26 +662,33 @@ bool perf_session__has_traces(struct perf_session *self, const char *msg)
         return true;
 }
 
-int perf_session__set_kallsyms_ref_reloc_sym(struct perf_session *self,
+int perf_session__set_kallsyms_ref_reloc_sym(struct map **maps,
                                              const char *symbol_name,
                                              u64 addr)
 {
         char *bracket;
         enum map_type i;
+        struct ref_reloc_sym *ref;
+
+        ref = zalloc(sizeof(struct ref_reloc_sym));
+        if (ref == NULL)
+                return -ENOMEM;
 
-        self->ref_reloc_sym.name = strdup(symbol_name);
-        if (self->ref_reloc_sym.name == NULL)
+        ref->name = strdup(symbol_name);
+        if (ref->name == NULL) {
+                free(ref);
                 return -ENOMEM;
+        }
 
-        bracket = strchr(self->ref_reloc_sym.name, ']');
+        bracket = strchr(ref->name, ']');
         if (bracket)
                 *bracket = '\0';
 
-        self->ref_reloc_sym.addr = addr;
+        ref->addr = addr;
 
         for (i = 0; i < MAP__NR_TYPES; ++i) {
-                struct kmap *kmap = map__kmap(self->vmlinux_maps[i]);
-                kmap->ref_reloc_sym = &self->ref_reloc_sym;
+                struct kmap *kmap = map__kmap(maps[i]);
+                kmap->ref_reloc_sym = ref;
         }
 
         return 0;
diff --git a/tools/perf/util/session.h b/tools/perf/util/session.h
index 0ac14d42dc28..5e47c87b9266 100644
--- a/tools/perf/util/session.h
+++ b/tools/perf/util/session.h
@@ -15,17 +15,15 @@ struct perf_session {
         struct perf_header      header;
         unsigned long           size;
         unsigned long           mmap_window;
-        struct map_groups       kmaps;
         struct rb_root          threads;
         struct thread           *last_match;
-        struct map              *vmlinux_maps[MAP__NR_TYPES];
+        struct rb_root          kerninfo_root;
         struct events_stats     events_stats;
         struct rb_root          stats_by_id;
         unsigned long           event_total[PERF_RECORD_MAX];
         unsigned long           unknown_events;
         struct rb_root          hists;
         u64                     sample_type;
-        struct ref_reloc_sym    ref_reloc_sym;
         int                     fd;
         bool                    fd_pipe;
         int                     cwdlen;
@@ -69,33 +67,13 @@ struct map_symbol *perf_session__resolve_callchain(struct perf_session *self,
 
 bool perf_session__has_traces(struct perf_session *self, const char *msg);
 
-int perf_header__read_build_ids(struct perf_header *self, int input,
-                                u64 offset, u64 file_size);
-
-int perf_session__set_kallsyms_ref_reloc_sym(struct perf_session *self,
+int perf_session__set_kallsyms_ref_reloc_sym(struct map **maps,
                                              const char *symbol_name,
                                              u64 addr);
 
 void mem_bswap_64(void *src, int byte_size);
 
-static inline int __perf_session__create_kernel_maps(struct perf_session *self,
-                                                struct dso *kernel)
-{
-        return __map_groups__create_kernel_maps(&self->kmaps,
-                                                self->vmlinux_maps, kernel);
-}
-
-static inline int perf_session__create_kernel_maps(struct perf_session *self)
-{
-        return map_groups__create_kernel_maps(&self->kmaps, self->vmlinux_maps);
-}
-
-static inline struct map *
-        perf_session__new_module_map(struct perf_session *self,
-                                     u64 start, const char *filename)
-{
-        return map_groups__new_module(&self->kmaps, start, filename);
-}
+int perf_session__create_kernel_maps(struct perf_session *self);
 
 int do_read(int fd, void *buf, size_t size);
 void perf_session__update_sample_type(struct perf_session *self);
diff --git a/tools/perf/util/sort.h b/tools/perf/util/sort.h
index 1d857aa2c01f..b7c54eeed9c9 100644
--- a/tools/perf/util/sort.h
+++ b/tools/perf/util/sort.h
@@ -44,6 +44,11 @@ extern enum sort_type sort__first_dimension;
 struct hist_entry {
         struct rb_node          rb_node;
         u64                     count;
+        u64                     count_sys;
+        u64                     count_us;
+        u64                     count_guest_sys;
+        u64                     count_guest_us;
+
         /*
          * XXX WARNING!
          * thread _has_ to come after ms, see
diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c
index f3d4151e46a1..e782e7db16c5 100644
--- a/tools/perf/util/symbol.c
+++ b/tools/perf/util/symbol.c
@@ -28,6 +28,8 @@ static void dsos__add(struct list_head *head, struct dso *dso);
 static struct map *map__new2(u64 start, struct dso *dso, enum map_type type);
 static int dso__load_kernel_sym(struct dso *self, struct map *map,
                                 symbol_filter_t filter);
+static int dso__load_guest_kernel_sym(struct dso *self, struct map *map,
+                        symbol_filter_t filter);
 static int vmlinux_path__nr_entries;
 static char **vmlinux_path;
 
@@ -186,6 +188,7 @@ struct dso *dso__new(const char *name)
                 self->loaded = 0;
                 self->sorted_by_name = 0;
                 self->has_build_id = 0;
+                self->kernel = DSO_TYPE_USER;
         }
 
         return self;
@@ -402,12 +405,9 @@ int kallsyms__parse(const char *filename, void *arg,
                 char *symbol_name;
 
                 line_len = getline(&line, &n, file);
-                if (line_len < 0)
+                if (line_len < 0 || !line)
                         break;
 
-                if (!line)
-                        goto out_failure;
-
                 line[--line_len] = '\0'; /* \n */
 
                 len = hex2u64(line, &start);
@@ -459,6 +459,7 @@ static int map__process_kallsym_symbol(void *arg, const char *name,
          * map__split_kallsyms, when we have split the maps per module
          */
         symbols__insert(root, sym);
+
         return 0;
 }
 
@@ -483,6 +484,7 @@ static int dso__split_kallsyms(struct dso *self, struct map *map,
                                symbol_filter_t filter)
 {
         struct map_groups *kmaps = map__kmap(map)->kmaps;
+        struct kernel_info *kerninfo = kmaps->this_kerninfo;
         struct map *curr_map = map;
         struct symbol *pos;
         int count = 0;
@@ -504,15 +506,33 @@ static int dso__split_kallsyms(struct dso *self, struct map *map,
                         *module++ = '\0';
 
                         if (strcmp(curr_map->dso->short_name, module)) {
-                                curr_map = map_groups__find_by_name(kmaps, map->type, module);
+                                if (curr_map != map &&
+                                        self->kernel == DSO_TYPE_GUEST_KERNEL &&
+                                        is_default_guest(kerninfo)) {
+                                        /*
+                                         * We assume all symbols of a module are
+                                         * continuous in * kallsyms, so curr_map
+                                         * points to a module and all its
+                                         * symbols are in its kmap. Mark it as
+                                         * loaded.
+                                         */
+                                        dso__set_loaded(curr_map->dso,
+                                                        curr_map->type);
+                                }
+
+                                curr_map = map_groups__find_by_name(kmaps,
+                                                        map->type, module);
                                 if (curr_map == NULL) {
-                                        pr_debug("/proc/{kallsyms,modules} "
+                                        pr_err("%s/proc/{kallsyms,modules} "
                                                  "inconsistency while looking "
-                                                 "for \"%s\" module!\n", module);
-                                        return -1;
+                                                 "for \"%s\" module!\n",
+                                                 kerninfo->root_dir, module);
+                                        curr_map = map;
+                                        goto discard_symbol;
                                 }
 
-                                if (curr_map->dso->loaded)
+                                if (curr_map->dso->loaded &&
+                                        !is_default_guest(kmaps->this_kerninfo))
                                         goto discard_symbol;
                         }
                         /*
@@ -525,13 +545,21 @@ static int dso__split_kallsyms(struct dso *self, struct map *map,
                         char dso_name[PATH_MAX];
                         struct dso *dso;
 
-                        snprintf(dso_name, sizeof(dso_name), "[kernel].%d",
-                                 kernel_range++);
+                        if (self->kernel == DSO_TYPE_GUEST_KERNEL)
+                                snprintf(dso_name, sizeof(dso_name),
+                                        "[guest.kernel].%d",
+                                        kernel_range++);
+                        else
+                                snprintf(dso_name, sizeof(dso_name),
+                                        "[kernel].%d",
+                                        kernel_range++);
 
                         dso = dso__new(dso_name);
                         if (dso == NULL)
                                 return -1;
 
+                        dso->kernel = self->kernel;
+
                         curr_map = map__new2(pos->start, dso, map->type);
                         if (curr_map == NULL) {
                                 dso__delete(dso);
@@ -555,6 +583,12 @@ discard_symbol:		rb_erase(&pos->rb_node, root);
                 }
         }
 
+        if (curr_map != map &&
+            self->kernel == DSO_TYPE_GUEST_KERNEL &&
+            is_default_guest(kmaps->this_kerninfo)) {
+                dso__set_loaded(curr_map->dso, curr_map->type);
+        }
+
         return count;
 }
 
@@ -565,7 +599,10 @@ int dso__load_kallsyms(struct dso *self, const char *filename,
                 return -1;
 
         symbols__fixup_end(&self->symbols[map->type]);
-        self->origin = DSO__ORIG_KERNEL;
+        if (self->kernel == DSO_TYPE_GUEST_KERNEL)
+                self->origin = DSO__ORIG_GUEST_KERNEL;
+        else
+                self->origin = DSO__ORIG_KERNEL;
 
         return dso__split_kallsyms(self, map, filter);
 }
@@ -952,7 +989,7 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name,
         nr_syms = shdr.sh_size / shdr.sh_entsize;
 
         memset(&sym, 0, sizeof(sym));
-        if (!self->kernel) {
+        if (self->kernel == DSO_TYPE_USER) {
                 self->adjust_symbols = (ehdr.e_type == ET_EXEC ||
                                 elf_section_by_name(elf, &ehdr, &shdr,
                                                      ".gnu.prelink_undo",
@@ -984,7 +1021,7 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name,
 
                 section_name = elf_sec__name(&shdr, secstrs);
 
-                if (self->kernel || kmodule) {
+                if (self->kernel != DSO_TYPE_USER || kmodule) {
                         char dso_name[PATH_MAX];
 
                         if (strcmp(section_name,
@@ -1011,6 +1048,7 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name,
                                 curr_dso = dso__new(dso_name);
                                 if (curr_dso == NULL)
                                         goto out_elf_end;
+                                curr_dso->kernel = self->kernel;
                                 curr_map = map__new2(start, curr_dso,
                                                      map->type);
                                 if (curr_map == NULL) {
@@ -1021,7 +1059,7 @@ static int dso__load_sym(struct dso *self, struct map *map, const char *name,
                                 curr_map->unmap_ip = identity__map_ip;
                                 curr_dso->origin = self->origin;
                                 map_groups__insert(kmap->kmaps, curr_map);
-                                dsos__add(&dsos__kernel, curr_dso);
+                                dsos__add(&self->node, curr_dso);
                                 dso__set_loaded(curr_dso, map->type);
                         } else
                                 curr_dso = curr_map->dso;
@@ -1083,7 +1121,7 @@ static bool dso__build_id_equal(const struct dso *self, u8 *build_id)
         return memcmp(self->build_id, build_id, sizeof(self->build_id)) == 0;
 }
 
-static bool __dsos__read_build_ids(struct list_head *head, bool with_hits)
+bool __dsos__read_build_ids(struct list_head *head, bool with_hits)
 {
         bool have_build_id = false;
         struct dso *pos;
@@ -1101,13 +1139,6 @@ static bool __dsos__read_build_ids(struct list_head *head, bool with_hits)
         return have_build_id;
 }
 
-bool dsos__read_build_ids(bool with_hits)
-{
-        bool kbuildids = __dsos__read_build_ids(&dsos__kernel, with_hits),
-             ubuildids = __dsos__read_build_ids(&dsos__user, with_hits);
-        return kbuildids || ubuildids;
-}
-
 /*
  * Align offset to 4 bytes as needed for note name and descriptor data.
  */
@@ -1242,6 +1273,8 @@ char dso__symtab_origin(const struct dso *self)
                 [DSO__ORIG_BUILDID] =  'b',
                 [DSO__ORIG_DSO] =      'd',
                 [DSO__ORIG_KMODULE] =  'K',
+                [DSO__ORIG_GUEST_KERNEL] =  'g',
+                [DSO__ORIG_GUEST_KMODULE] =  'G',
         };
 
         if (self == NULL || self->origin == DSO__ORIG_NOT_FOUND)
@@ -1257,11 +1290,20 @@ int dso__load(struct dso *self, struct map *map, symbol_filter_t filter)
         char build_id_hex[BUILD_ID_SIZE * 2 + 1];
         int ret = -1;
         int fd;
+        struct kernel_info *kerninfo;
+        const char *root_dir;
 
         dso__set_loaded(self, map->type);
 
-        if (self->kernel)
+        if (self->kernel == DSO_TYPE_KERNEL)
                 return dso__load_kernel_sym(self, map, filter);
+        else if (self->kernel == DSO_TYPE_GUEST_KERNEL)
+                return dso__load_guest_kernel_sym(self, map, filter);
+
+        if (map->groups && map->groups->this_kerninfo)
+                kerninfo = map->groups->this_kerninfo;
+        else
+                kerninfo = NULL;
 
         name = malloc(size);
         if (!name)
@@ -1315,6 +1357,13 @@ more:
                 case DSO__ORIG_DSO:
                         snprintf(name, size, "%s", self->long_name);
                         break;
+                case DSO__ORIG_GUEST_KMODULE:
+                        if (map->groups && map->groups->this_kerninfo)
+                                root_dir = map->groups->this_kerninfo->root_dir;
+                        else
+                                root_dir = "";
+                        snprintf(name, size, "%s%s", root_dir, self->long_name);
+                        break;
 
                 default:
                         goto out;
@@ -1368,7 +1417,8 @@ struct map *map_groups__find_by_name(struct map_groups *self,
         return NULL;
 }
 
-static int dso__kernel_module_get_build_id(struct dso *self)
+static int dso__kernel_module_get_build_id(struct dso *self,
+                                const char *root_dir)
 {
         char filename[PATH_MAX];
         /*
@@ -1378,8 +1428,8 @@ static int dso__kernel_module_get_build_id(struct dso *self)
         const char *name = self->short_name + 1;
 
         snprintf(filename, sizeof(filename),
-                 "/sys/module/%.*s/notes/.note.gnu.build-id",
-                 (int)strlen(name - 1), name);
+                 "%s/sys/module/%.*s/notes/.note.gnu.build-id",
+                 root_dir, (int)strlen(name) - 1, name);
 
         if (sysfs__read_build_id(filename, self->build_id,
                                  sizeof(self->build_id)) == 0)
@@ -1388,7 +1438,8 @@ static int dso__kernel_module_get_build_id(struct dso *self)
         return 0;
 }
 
-static int map_groups__set_modules_path_dir(struct map_groups *self, char *dir_name)
+static int map_groups__set_modules_path_dir(struct map_groups *self,
+                                const char *dir_name)
 {
         struct dirent *dent;
         DIR *dir = opendir(dir_name);
@@ -1400,8 +1451,14 @@ static int map_groups__set_modules_path_dir(struct map_groups *self, char *dir_n
 
         while ((dent = readdir(dir)) != NULL) {
                 char path[PATH_MAX];
+                struct stat st;
+
+                /*sshfs might return bad dent->d_type, so we have to stat*/
+                sprintf(path, "%s/%s", dir_name, dent->d_name);
+                if (stat(path, &st))
+                        continue;
 
-                if (dent->d_type == DT_DIR) {
+                if (S_ISDIR(st.st_mode)) {
                         if (!strcmp(dent->d_name, ".") ||
                             !strcmp(dent->d_name, ".."))
                                 continue;
@@ -1433,7 +1490,7 @@ static int map_groups__set_modules_path_dir(struct map_groups *self, char *dir_n
                         if (long_name == NULL)
                                 goto failure;
                         dso__set_long_name(map->dso, long_name);
-                        dso__kernel_module_get_build_id(map->dso);
+                        dso__kernel_module_get_build_id(map->dso, "");
                 }
         }
 
@@ -1443,16 +1500,46 @@ failure:
         return -1;
 }
 
-static int map_groups__set_modules_path(struct map_groups *self)
+static char *get_kernel_version(const char *root_dir)
 {
-        struct utsname uts;
+        char version[PATH_MAX];
+        FILE *file;
+        char *name, *tmp;
+        const char *prefix = "Linux version ";
+
+        sprintf(version, "%s/proc/version", root_dir);
+        file = fopen(version, "r");
+        if (!file)
+                return NULL;
+
+        version[0] = '\0';
+        tmp = fgets(version, sizeof(version), file);
+        fclose(file);
+
+        name = strstr(version, prefix);
+        if (!name)
+                return NULL;
+        name += strlen(prefix);
+        tmp = strchr(name, ' ');
+        if (tmp)
+                *tmp = '\0';
+
+        return strdup(name);
+}
+
+static int map_groups__set_modules_path(struct map_groups *self,
+                                const char *root_dir)
+{
+        char *version;
         char modules_path[PATH_MAX];
 
-        if (uname(&uts) < 0)
+        version = get_kernel_version(root_dir);
+        if (!version)
                 return -1;
 
-        snprintf(modules_path, sizeof(modules_path), "/lib/modules/%s/kernel",
-                 uts.release);
+        snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s/kernel",
+                 root_dir, version);
+        free(version);
 
         return map_groups__set_modules_path_dir(self, modules_path);
 }
@@ -1477,11 +1564,13 @@ static struct map *map__new2(u64 start, struct dso *dso, enum map_type type)
 }
 
 struct map *map_groups__new_module(struct map_groups *self, u64 start,
-                                   const char *filename)
+                                const char *filename,
+                                struct kernel_info *kerninfo)
 {
         struct map *map;
-        struct dso *dso = __dsos__findnew(&dsos__kernel, filename);
+        struct dso *dso;
 
+        dso = __dsos__findnew(&kerninfo->dsos__kernel, filename);
         if (dso == NULL)
                 return NULL;
 
@@ -1489,21 +1578,37 @@ struct map *map_groups__new_module(struct map_groups *self, u64 start,
         if (map == NULL)
                 return NULL;
 
-        dso->origin = DSO__ORIG_KMODULE;
+        if (is_host_kernel(kerninfo))
+                dso->origin = DSO__ORIG_KMODULE;
+        else
+                dso->origin = DSO__ORIG_GUEST_KMODULE;
         map_groups__insert(self, map);
         return map;
 }
 
-static int map_groups__create_modules(struct map_groups *self)
+static int map_groups__create_modules(struct kernel_info *kerninfo)
 {
         char *line = NULL;
         size_t n;
-        FILE *file = fopen("/proc/modules", "r");
+        FILE *file;
         struct map *map;
+        const char *root_dir;
+        const char *modules;
+        char path[PATH_MAX];
+
+        if (is_default_guest(kerninfo))
+                modules = symbol_conf.default_guest_modules;
+        else {
+                sprintf(path, "%s/proc/modules", kerninfo->root_dir);
+                modules = path;
+        }
 
+        file = fopen(modules, "r");
         if (file == NULL)
                 return -1;
 
+        root_dir = kerninfo->root_dir;
+
         while (!feof(file)) {
                 char name[PATH_MAX];
                 u64 start;
@@ -1532,16 +1637,17 @@ static int map_groups__create_modules(struct map_groups *self)
                 *sep = '\0';
 
                 snprintf(name, sizeof(name), "[%s]", line);
-                map = map_groups__new_module(self, start, name);
+                map = map_groups__new_module(&kerninfo->kmaps,
+                                start, name, kerninfo);
                 if (map == NULL)
                         goto out_delete_line;
-                dso__kernel_module_get_build_id(map->dso);
+                dso__kernel_module_get_build_id(map->dso, root_dir);
         }
 
         free(line);
         fclose(file);
 
-        return map_groups__set_modules_path(self);
+        return map_groups__set_modules_path(&kerninfo->kmaps, root_dir);
 
 out_delete_line:
         free(line);
@@ -1708,8 +1814,57 @@ out_fixup:
         return err;
 }
 
-LIST_HEAD(dsos__user);
-LIST_HEAD(dsos__kernel);
+static int dso__load_guest_kernel_sym(struct dso *self, struct map *map,
+                                symbol_filter_t filter)
+{
+        int err;
+        const char *kallsyms_filename = NULL;
+        struct kernel_info *kerninfo;
+        char path[PATH_MAX];
+
+        if (!map->groups) {
+                pr_debug("Guest kernel map hasn't the point to groups\n");
+                return -1;
+        }
+        kerninfo = map->groups->this_kerninfo;
+
+        if (is_default_guest(kerninfo)) {
+                /*
+                 * if the user specified a vmlinux filename, use it and only
+                 * it, reporting errors to the user if it cannot be used.
+                 * Or use file guest_kallsyms inputted by user on commandline
+                 */
+                if (symbol_conf.default_guest_vmlinux_name != NULL) {
+                        err = dso__load_vmlinux(self, map,
+                                symbol_conf.default_guest_vmlinux_name, filter);
+                        goto out_try_fixup;
+                }
+
+                kallsyms_filename = symbol_conf.default_guest_kallsyms;
+                if (!kallsyms_filename)
+                        return -1;
+        } else {
+                sprintf(path, "%s/proc/kallsyms", kerninfo->root_dir);
+                kallsyms_filename = path;
+        }
+
+        err = dso__load_kallsyms(self, kallsyms_filename, map, filter);
+        if (err > 0)
+                pr_debug("Using %s for symbols\n", kallsyms_filename);
+
+out_try_fixup:
+        if (err > 0) {
+                if (kallsyms_filename != NULL) {
+                        kern_mmap_name(kerninfo, path);
+                        dso__set_long_name(self,
+                                strdup(path));
+                }
+                map__fixup_start(map);
+                map__fixup_end(map);
+        }
+
+        return err;
+}
 
 static void dsos__add(struct list_head *head, struct dso *dso)
 {
@@ -1752,10 +1907,16 @@ static void __dsos__fprintf(struct list_head *head, FILE *fp)
         }
 }
 
-void dsos__fprintf(FILE *fp)
+void dsos__fprintf(struct rb_root *kerninfo_root, FILE *fp)
 {
-        __dsos__fprintf(&dsos__kernel, fp);
-        __dsos__fprintf(&dsos__user, fp);
+        struct rb_node *nd;
+
+        for (nd = rb_first(kerninfo_root); nd; nd = rb_next(nd)) {
+                struct kernel_info *pos = rb_entry(nd, struct kernel_info,
+                                rb_node);
+                __dsos__fprintf(&pos->dsos__kernel, fp);
+                __dsos__fprintf(&pos->dsos__user, fp);
+        }
 }
 
 static size_t __dsos__fprintf_buildid(struct list_head *head, FILE *fp,
@@ -1773,10 +1934,21 @@ static size_t __dsos__fprintf_buildid(struct list_head *head, FILE *fp,
         return ret;
 }
 
-size_t dsos__fprintf_buildid(FILE *fp, bool with_hits)
+size_t dsos__fprintf_buildid(struct rb_root *kerninfo_root,
+                FILE *fp, bool with_hits)
 {
-        return (__dsos__fprintf_buildid(&dsos__kernel, fp, with_hits) +
-                __dsos__fprintf_buildid(&dsos__user, fp, with_hits));
+        struct rb_node *nd;
+        size_t ret = 0;
+
+        for (nd = rb_first(kerninfo_root); nd; nd = rb_next(nd)) {
+                struct kernel_info *pos = rb_entry(nd, struct kernel_info,
+                                rb_node);
+                ret += __dsos__fprintf_buildid(&pos->dsos__kernel,
+                                        fp, with_hits);
+                ret += __dsos__fprintf_buildid(&pos->dsos__user,
+                                        fp, with_hits);
+        }
+        return ret;
 }
 
 struct dso *dso__new_kernel(const char *name)
@@ -1785,28 +1957,59 @@ struct dso *dso__new_kernel(const char *name)
 
         if (self != NULL) {
                 dso__set_short_name(self, "[kernel]");
-                self->kernel     = 1;
+                self->kernel = DSO_TYPE_KERNEL;
+        }
+
+        return self;
+}
+
+static struct dso *dso__new_guest_kernel(struct kernel_info *kerninfo,
+                                        const char *name)
+{
+        char buff[PATH_MAX];
+        struct dso *self;
+
+        kern_mmap_name(kerninfo, buff);
+        self = dso__new(name ?: buff);
+        if (self != NULL) {
+                dso__set_short_name(self, "[guest.kernel]");
+                self->kernel = DSO_TYPE_GUEST_KERNEL;
         }
 
         return self;
 }
 
-void dso__read_running_kernel_build_id(struct dso *self)
+void dso__read_running_kernel_build_id(struct dso *self,
+                        struct kernel_info *kerninfo)
 {
-        if (sysfs__read_build_id("/sys/kernel/notes", self->build_id,
+        char path[PATH_MAX];
+
+        if (is_default_guest(kerninfo))
+                return;
+        sprintf(path, "%s/sys/kernel/notes", kerninfo->root_dir);
+        if (sysfs__read_build_id(path, self->build_id,
                                  sizeof(self->build_id)) == 0)
                 self->has_build_id = true;
 }
 
-static struct dso *dsos__create_kernel(const char *vmlinux)
+static struct dso *dsos__create_kernel(struct kernel_info *kerninfo)
 {
-        struct dso *kernel = dso__new_kernel(vmlinux);
+        const char *vmlinux_name = NULL;
+        struct dso *kernel;
 
-        if (kernel != NULL) {
-                dso__read_running_kernel_build_id(kernel);
-                dsos__add(&dsos__kernel, kernel);
+        if (is_host_kernel(kerninfo)) {
+                vmlinux_name = symbol_conf.vmlinux_name;
+                kernel = dso__new_kernel(vmlinux_name);
+        } else {
+                if (is_default_guest(kerninfo))
+                        vmlinux_name = symbol_conf.default_guest_vmlinux_name;
+                kernel = dso__new_guest_kernel(kerninfo, vmlinux_name);
         }
 
+        if (kernel != NULL) {
+                dso__read_running_kernel_build_id(kernel, kerninfo);
+                dsos__add(&kerninfo->dsos__kernel, kernel);
+        }
         return kernel;
 }
 
@@ -1950,23 +2153,29 @@ out_free_comm_list:
         return -1;
 }
 
-int map_groups__create_kernel_maps(struct map_groups *self,
-                                   struct map *vmlinux_maps[MAP__NR_TYPES])
+int map_groups__create_kernel_maps(struct rb_root *kerninfo_root, pid_t pid)
 {
-        struct dso *kernel = dsos__create_kernel(symbol_conf.vmlinux_name);
+        struct kernel_info *kerninfo;
+        struct dso *kernel;
 
+        kerninfo = kerninfo__findnew(kerninfo_root, pid);
+        if (kerninfo == NULL)
+                return -1;
+        kernel = dsos__create_kernel(kerninfo);
         if (kernel == NULL)
                 return -1;
 
-        if (__map_groups__create_kernel_maps(self, vmlinux_maps, kernel) < 0)
+        if (__map_groups__create_kernel_maps(&kerninfo->kmaps,
+                        kerninfo->vmlinux_maps, kernel) < 0)
                 return -1;
 
-        if (symbol_conf.use_modules && map_groups__create_modules(self) < 0)
+        if (symbol_conf.use_modules &&
+                map_groups__create_modules(kerninfo) < 0)
                 pr_debug("Problems creating module maps, continuing anyway...\n");
         /*
          * Now that we have all the maps created, just set the ->end of them:
          */
-        map_groups__fixup_end(self);
+        map_groups__fixup_end(&kerninfo->kmaps);
         return 0;
 }
 
@@ -2012,3 +2221,46 @@ char *strxfrchar(char *s, char from, char to)
 
         return s;
 }
+
+int map_groups__create_guest_kernel_maps(struct rb_root *kerninfo_root)
+{
+        int ret = 0;
+        struct dirent **namelist = NULL;
+        int i, items = 0;
+        char path[PATH_MAX];
+        pid_t pid;
+
+        if (symbol_conf.default_guest_vmlinux_name ||
+            symbol_conf.default_guest_modules ||
+            symbol_conf.default_guest_kallsyms) {
+                map_groups__create_kernel_maps(kerninfo_root,
+                                        DEFAULT_GUEST_KERNEL_ID);
+        }
+
+        if (symbol_conf.guestmount) {
+                items = scandir(symbol_conf.guestmount, &namelist, NULL, NULL);
+                if (items <= 0)
+                        return -ENOENT;
+                for (i = 0; i < items; i++) {
+                        if (!isdigit(namelist[i]->d_name[0])) {
+                                /* Filter out . and .. */
+                                continue;
+                        }
+                        pid = atoi(namelist[i]->d_name);
+                        sprintf(path, "%s/%s/proc/kallsyms",
+                                symbol_conf.guestmount,
+                                namelist[i]->d_name);
+                        ret = access(path, R_OK);
+                        if (ret) {
+                                pr_debug("Can't access file %s\n", path);
+                                goto failure;
+                        }
+                        map_groups__create_kernel_maps(kerninfo_root,
+                                                        pid);
+                }
+failure:
+                free(namelist);
+        }
+
+        return ret;
+}
diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h
index 757fae3f5ee0..478f5ab37787 100644
--- a/tools/perf/util/symbol.h
+++ b/tools/perf/util/symbol.h
@@ -69,10 +69,15 @@ struct symbol_conf {
                         show_nr_samples,
                         use_callchain,
                         exclude_other,
-                        full_paths;
+                        full_paths,
+                        show_cpu_utilization;
         const char      *vmlinux_name,
                         *field_sep;
-        char            *dso_list_str,
+        const char      *default_guest_vmlinux_name,
+                        *default_guest_kallsyms,
+                        *default_guest_modules;
+        const char      *guestmount;
+        char            *dso_list_str,
                         *comm_list_str,
                         *sym_list_str,
                         *col_width_list_str;
@@ -106,6 +111,13 @@ struct addr_location {
         u64           addr;
         char          level;
         bool          filtered;
+        unsigned int  cpumode;
+};
+
+enum dso_kernel_type {
+        DSO_TYPE_USER = 0,
+        DSO_TYPE_KERNEL,
+        DSO_TYPE_GUEST_KERNEL
 };
 
 struct dso {
@@ -115,7 +127,7 @@ struct dso {
         u8               adjust_symbols:1;
         u8               slen_calculated:1;
         u8               has_build_id:1;
-        u8               kernel:1;
+        enum dso_kernel_type    kernel;
         u8               hit:1;
         u8               annotate_warned:1;
         unsigned char    origin;
@@ -143,34 +155,30 @@ static inline void dso__set_loaded(struct dso *self, enum map_type type)
 
 void dso__sort_by_name(struct dso *self, enum map_type type);
 
-extern struct list_head dsos__user, dsos__kernel;
-
 struct dso *__dsos__findnew(struct list_head *head, const char *name);
 
-static inline struct dso *dsos__findnew(const char *name)
-{
-        return __dsos__findnew(&dsos__user, name);
-}
-
 int dso__load(struct dso *self, struct map *map, symbol_filter_t filter);
 int dso__load_vmlinux_path(struct dso *self, struct map *map,
                            symbol_filter_t filter);
 int dso__load_kallsyms(struct dso *self, const char *filename, struct map *map,
                        symbol_filter_t filter);
-void dsos__fprintf(FILE *fp);
-size_t dsos__fprintf_buildid(FILE *fp, bool with_hits);
+void dsos__fprintf(struct rb_root *kerninfo_root, FILE *fp);
+size_t dsos__fprintf_buildid(struct rb_root *kerninfo_root,
+                FILE *fp, bool with_hits);
 
 size_t dso__fprintf_buildid(struct dso *self, FILE *fp);
 size_t dso__fprintf(struct dso *self, enum map_type type, FILE *fp);
 
 enum dso_origin {
         DSO__ORIG_KERNEL = 0,
+        DSO__ORIG_GUEST_KERNEL,
         DSO__ORIG_JAVA_JIT,
         DSO__ORIG_BUILD_ID_CACHE,
         DSO__ORIG_FEDORA,
         DSO__ORIG_UBUNTU,
         DSO__ORIG_BUILDID,
         DSO__ORIG_DSO,
+        DSO__ORIG_GUEST_KMODULE,
         DSO__ORIG_KMODULE,
         DSO__ORIG_NOT_FOUND,
 };
@@ -178,19 +186,26 @@ enum dso_origin {
 char dso__symtab_origin(const struct dso *self);
 void dso__set_long_name(struct dso *self, char *name);
 void dso__set_build_id(struct dso *self, void *build_id);
-void dso__read_running_kernel_build_id(struct dso *self);
+void dso__read_running_kernel_build_id(struct dso *self,
+                struct kernel_info *kerninfo);
 struct symbol *dso__find_symbol(struct dso *self, enum map_type type, u64 addr);
 struct symbol *dso__find_symbol_by_name(struct dso *self, enum map_type type,
                                         const char *name);
 
 int filename__read_build_id(const char *filename, void *bf, size_t size);
 int sysfs__read_build_id(const char *filename, void *bf, size_t size);
-bool dsos__read_build_ids(bool with_hits);
+bool __dsos__read_build_ids(struct list_head *head, bool with_hits);
 int build_id__sprintf(const u8 *self, int len, char *bf);
 int kallsyms__parse(const char *filename, void *arg,
                     int (*process_symbol)(void *arg, const char *name,
                                           char type, u64 start));
 
+int __map_groups__create_kernel_maps(struct map_groups *self,
+                        struct map *vmlinux_maps[MAP__NR_TYPES],
+                        struct dso *kernel);
+int map_groups__create_kernel_maps(struct rb_root *kerninfo_root, pid_t pid);
+int map_groups__create_guest_kernel_maps(struct rb_root *kerninfo_root);
+
 int symbol__init(void);
 bool symbol_type__is_a(char symbol_type, enum map_type map_type);
 
diff --git a/tools/perf/util/thread.h b/tools/perf/util/thread.h
index 9c488fcadec9..1dfd9ff8bdcd 100644
--- a/tools/perf/util/thread.h
+++ b/tools/perf/util/thread.h
@@ -33,12 +33,12 @@ static inline struct map *thread__find_map(struct thread *self,
 
 void thread__find_addr_map(struct thread *self,
                            struct perf_session *session, u8 cpumode,
-                           enum map_type type, u64 addr,
+                           enum map_type type, pid_t pid, u64 addr,
                            struct addr_location *al);
 
 void thread__find_addr_location(struct thread *self,
                                 struct perf_session *session, u8 cpumode,
-                                enum map_type type, u64 addr,
+                                enum map_type type, pid_t pid, u64 addr,
                                 struct addr_location *al,
                                 symbol_filter_t filter);
 #endif  /* __PERF_THREAD_H */