1 files changed, 258 insertions, 168 deletions

diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c
index 855e35cbb1dc..594d14a02b67 100644
--- a/tools/perf/util/stat-shadow.c
+++ b/tools/perf/util/stat-shadow.c
@@ -9,17 +9,6 @@
 #include "expr.h"
 #include "metricgroup.h"
 
-enum {
-        CTX_BIT_USER    = 1 << 0,
-        CTX_BIT_KERNEL  = 1 << 1,
-        CTX_BIT_HV      = 1 << 2,
-        CTX_BIT_HOST    = 1 << 3,
-        CTX_BIT_IDLE    = 1 << 4,
-        CTX_BIT_MAX     = 1 << 5,
-};
-
-#define NUM_CTX CTX_BIT_MAX
-
 /*
  * AGGR_GLOBAL: Use CPU 0
  * AGGR_SOCKET: Use first CPU of socket
@@ -27,36 +16,18 @@ enum {
  * AGGR_NONE: Use matching CPU
  * AGGR_THREAD: Not supported?
  */
-static struct stats runtime_nsecs_stats[MAX_NR_CPUS];
-static struct stats runtime_cycles_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_stalled_cycles_front_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_stalled_cycles_back_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_branches_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_cacherefs_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_l1_dcache_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_l1_icache_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_ll_cache_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_itlb_cache_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_dtlb_cache_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_cycles_in_tx_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_transaction_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_elision_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_topdown_total_slots[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_topdown_slots_issued[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_topdown_slots_retired[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_topdown_fetch_bubbles[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_topdown_recovery_bubbles[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_smi_num_stats[NUM_CTX][MAX_NR_CPUS];
-static struct stats runtime_aperf_stats[NUM_CTX][MAX_NR_CPUS];
-static struct rblist runtime_saved_values;
 static bool have_frontend_stalled;
 
+struct runtime_stat rt_stat;
 struct stats walltime_nsecs_stats;
 
 struct saved_value {
         struct rb_node rb_node;
         struct perf_evsel *evsel;
+        enum stat_type type;
+        int ctx;
         int cpu;
+        struct runtime_stat *stat;
         struct stats stats;
 };
 
@@ -69,6 +40,30 @@ static int saved_value_cmp(struct rb_node *rb_node, const void *entry)
 
         if (a->cpu != b->cpu)
                 return a->cpu - b->cpu;
+
+        /*
+         * Previously the rbtree was used to link generic metrics.
+         * The keys were evsel/cpu. Now the rbtree is extended to support
+         * per-thread shadow stats. For shadow stats case, the keys
+         * are cpu/type/ctx/stat (evsel is NULL). For generic metrics
+         * case, the keys are still evsel/cpu (type/ctx/stat are 0 or NULL).
+         */
+        if (a->type != b->type)
+                return a->type - b->type;
+
+        if (a->ctx != b->ctx)
+                return a->ctx - b->ctx;
+
+        if (a->evsel == NULL && b->evsel == NULL) {
+                if (a->stat == b->stat)
+                        return 0;
+
+                if ((char *)a->stat < (char *)b->stat)
+                        return -1;
+
+                return 1;
+        }
+
         if (a->evsel == b->evsel)
                 return 0;
         if ((char *)a->evsel < (char *)b->evsel)
@@ -87,34 +82,66 @@ static struct rb_node *saved_value_new(struct rblist *rblist __maybe_unused,
         return &nd->rb_node;
 }
 
+static void saved_value_delete(struct rblist *rblist __maybe_unused,
+                               struct rb_node *rb_node)
+{
+        struct saved_value *v;
+
+        BUG_ON(!rb_node);
+        v = container_of(rb_node, struct saved_value, rb_node);
+        free(v);
+}
+
 static struct saved_value *saved_value_lookup(struct perf_evsel *evsel,
                                               int cpu,
-                                              bool create)
+                                              bool create,
+                                              enum stat_type type,
+                                              int ctx,
+                                              struct runtime_stat *st)
 {
+        struct rblist *rblist;
         struct rb_node *nd;
         struct saved_value dm = {
                 .cpu = cpu,
                 .evsel = evsel,
+                .type = type,
+                .ctx = ctx,
+                .stat = st,
         };
-        nd = rblist__find(&runtime_saved_values, &dm);
+
+        rblist = &st->value_list;
+
+        nd = rblist__find(rblist, &dm);
         if (nd)
                 return container_of(nd, struct saved_value, rb_node);
         if (create) {
-                rblist__add_node(&runtime_saved_values, &dm);
-                nd = rblist__find(&runtime_saved_values, &dm);
+                rblist__add_node(rblist, &dm);
+                nd = rblist__find(rblist, &dm);
                 if (nd)
                         return container_of(nd, struct saved_value, rb_node);
         }
         return NULL;
 }
 
+void runtime_stat__init(struct runtime_stat *st)
+{
+        struct rblist *rblist = &st->value_list;
+
+        rblist__init(rblist);
+        rblist->node_cmp = saved_value_cmp;
+        rblist->node_new = saved_value_new;
+        rblist->node_delete = saved_value_delete;
+}
+
+void runtime_stat__exit(struct runtime_stat *st)
+{
+        rblist__exit(&st->value_list);
+}
+
 void perf_stat__init_shadow_stats(void)
 {
         have_frontend_stalled = pmu_have_event("cpu", "stalled-cycles-frontend");
-        rblist__init(&runtime_saved_values);
-        runtime_saved_values.node_cmp = saved_value_cmp;
-        runtime_saved_values.node_new = saved_value_new;
-        /* No delete for now */
+        runtime_stat__init(&rt_stat);
 }
 
 static int evsel_context(struct perf_evsel *evsel)
@@ -135,36 +162,13 @@ static int evsel_context(struct perf_evsel *evsel)
         return ctx;
 }
 
-void perf_stat__reset_shadow_stats(void)
+static void reset_stat(struct runtime_stat *st)
 {
+        struct rblist *rblist;
         struct rb_node *pos, *next;
 
-        memset(runtime_nsecs_stats, 0, sizeof(runtime_nsecs_stats));
-        memset(runtime_cycles_stats, 0, sizeof(runtime_cycles_stats));
-        memset(runtime_stalled_cycles_front_stats, 0, sizeof(runtime_stalled_cycles_front_stats));
-        memset(runtime_stalled_cycles_back_stats, 0, sizeof(runtime_stalled_cycles_back_stats));
-        memset(runtime_branches_stats, 0, sizeof(runtime_branches_stats));
-        memset(runtime_cacherefs_stats, 0, sizeof(runtime_cacherefs_stats));
-        memset(runtime_l1_dcache_stats, 0, sizeof(runtime_l1_dcache_stats));
-        memset(runtime_l1_icache_stats, 0, sizeof(runtime_l1_icache_stats));
-        memset(runtime_ll_cache_stats, 0, sizeof(runtime_ll_cache_stats));
-        memset(runtime_itlb_cache_stats, 0, sizeof(runtime_itlb_cache_stats));
-        memset(runtime_dtlb_cache_stats, 0, sizeof(runtime_dtlb_cache_stats));
-        memset(runtime_cycles_in_tx_stats, 0,
-                        sizeof(runtime_cycles_in_tx_stats));
-        memset(runtime_transaction_stats, 0,
-                sizeof(runtime_transaction_stats));
-        memset(runtime_elision_stats, 0, sizeof(runtime_elision_stats));
-        memset(&walltime_nsecs_stats, 0, sizeof(walltime_nsecs_stats));
-        memset(runtime_topdown_total_slots, 0, sizeof(runtime_topdown_total_slots));
-        memset(runtime_topdown_slots_retired, 0, sizeof(runtime_topdown_slots_retired));
-        memset(runtime_topdown_slots_issued, 0, sizeof(runtime_topdown_slots_issued));
-        memset(runtime_topdown_fetch_bubbles, 0, sizeof(runtime_topdown_fetch_bubbles));
-        memset(runtime_topdown_recovery_bubbles, 0, sizeof(runtime_topdown_recovery_bubbles));
-        memset(runtime_smi_num_stats, 0, sizeof(runtime_smi_num_stats));
-        memset(runtime_aperf_stats, 0, sizeof(runtime_aperf_stats));
-
-        next = rb_first(&runtime_saved_values.entries);
+        rblist = &st->value_list;
+        next = rb_first(&rblist->entries);
         while (next) {
                 pos = next;
                 next = rb_next(pos);
@@ -174,13 +178,35 @@ void perf_stat__reset_shadow_stats(void)
         }
 }
 
+void perf_stat__reset_shadow_stats(void)
+{
+        reset_stat(&rt_stat);
+        memset(&walltime_nsecs_stats, 0, sizeof(walltime_nsecs_stats));
+}
+
+void perf_stat__reset_shadow_per_stat(struct runtime_stat *st)
+{
+        reset_stat(st);
+}
+
+static void update_runtime_stat(struct runtime_stat *st,
+                                enum stat_type type,
+                                int ctx, int cpu, u64 count)
+{
+        struct saved_value *v = saved_value_lookup(NULL, cpu, true,
+                                                   type, ctx, st);
+
+        if (v)
+                update_stats(&v->stats, count);
+}
+
 /*
  * Update various tracking values we maintain to print
  * more semantic information such as miss/hit ratios,
  * instruction rates, etc:
  */
 void perf_stat__update_shadow_stats(struct perf_evsel *counter, u64 count,
-                                    int cpu)
+                                    int cpu, struct runtime_stat *st)
 {
         int ctx = evsel_context(counter);
 
@@ -188,50 +214,58 @@ void perf_stat__update_shadow_stats(struct perf_evsel *counter, u64 count,
 
         if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK) ||
             perf_evsel__match(counter, SOFTWARE, SW_CPU_CLOCK))
-                update_stats(&runtime_nsecs_stats[cpu], count);
+                update_runtime_stat(st, STAT_NSECS, 0, cpu, count);
         else if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES))
-                update_stats(&runtime_cycles_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_CYCLES, ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, CYCLES_IN_TX))
-                update_stats(&runtime_cycles_in_tx_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_CYCLES_IN_TX, ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, TRANSACTION_START))
-                update_stats(&runtime_transaction_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_TRANSACTION, ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, ELISION_START))
-                update_stats(&runtime_elision_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_ELISION, ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, TOPDOWN_TOTAL_SLOTS))
-                update_stats(&runtime_topdown_total_slots[ctx][cpu], count);
+                update_runtime_stat(st, STAT_TOPDOWN_TOTAL_SLOTS,
+                                    ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, TOPDOWN_SLOTS_ISSUED))
-                update_stats(&runtime_topdown_slots_issued[ctx][cpu], count);
+                update_runtime_stat(st, STAT_TOPDOWN_SLOTS_ISSUED,
+                                    ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, TOPDOWN_SLOTS_RETIRED))
-                update_stats(&runtime_topdown_slots_retired[ctx][cpu], count);
+                update_runtime_stat(st, STAT_TOPDOWN_SLOTS_RETIRED,
+                                    ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, TOPDOWN_FETCH_BUBBLES))
-                update_stats(&runtime_topdown_fetch_bubbles[ctx][cpu], count);
+                update_runtime_stat(st, STAT_TOPDOWN_FETCH_BUBBLES,
+                                    ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, TOPDOWN_RECOVERY_BUBBLES))
-                update_stats(&runtime_topdown_recovery_bubbles[ctx][cpu], count);
+                update_runtime_stat(st, STAT_TOPDOWN_RECOVERY_BUBBLES,
+                                    ctx, cpu, count);
         else if (perf_evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_FRONTEND))
-                update_stats(&runtime_stalled_cycles_front_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_STALLED_CYCLES_FRONT,
+                                    ctx, cpu, count);
         else if (perf_evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_BACKEND))
-                update_stats(&runtime_stalled_cycles_back_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_STALLED_CYCLES_BACK,
+                                    ctx, cpu, count);
         else if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS))
-                update_stats(&runtime_branches_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_BRANCHES, ctx, cpu, count);
         else if (perf_evsel__match(counter, HARDWARE, HW_CACHE_REFERENCES))
-                update_stats(&runtime_cacherefs_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_CACHEREFS, ctx, cpu, count);
         else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_L1D))
-                update_stats(&runtime_l1_dcache_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_L1_DCACHE, ctx, cpu, count);
         else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_L1I))
-                update_stats(&runtime_ll_cache_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_L1_ICACHE, ctx, cpu, count);
         else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_LL))
-                update_stats(&runtime_ll_cache_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_LL_CACHE, ctx, cpu, count);
         else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_DTLB))
-                update_stats(&runtime_dtlb_cache_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_DTLB_CACHE, ctx, cpu, count);
         else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_ITLB))
-                update_stats(&runtime_itlb_cache_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_ITLB_CACHE, ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, SMI_NUM))
-                update_stats(&runtime_smi_num_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_SMI_NUM, ctx, cpu, count);
         else if (perf_stat_evsel__is(counter, APERF))
-                update_stats(&runtime_aperf_stats[ctx][cpu], count);
+                update_runtime_stat(st, STAT_APERF, ctx, cpu, count);
 
         if (counter->collect_stat) {
-                struct saved_value *v = saved_value_lookup(counter, cpu, true);
+                struct saved_value *v = saved_value_lookup(counter, cpu, true,
+                                                           STAT_NONE, 0, st);
                 update_stats(&v->stats, count);
         }
 }
@@ -352,15 +386,40 @@ void perf_stat__collect_metric_expr(struct perf_evlist *evsel_list)
         }
 }
 
+static double runtime_stat_avg(struct runtime_stat *st,
+                               enum stat_type type, int ctx, int cpu)
+{
+        struct saved_value *v;
+
+        v = saved_value_lookup(NULL, cpu, false, type, ctx, st);
+        if (!v)
+                return 0.0;
+
+        return avg_stats(&v->stats);
+}
+
+static double runtime_stat_n(struct runtime_stat *st,
+                             enum stat_type type, int ctx, int cpu)
+{
+        struct saved_value *v;
+
+        v = saved_value_lookup(NULL, cpu, false, type, ctx, st);
+        if (!v)
+                return 0.0;
+
+        return v->stats.n;
+}
+
 static void print_stalled_cycles_frontend(int cpu,
                                           struct perf_evsel *evsel, double avg,
-                                          struct perf_stat_output_ctx *out)
+                                          struct perf_stat_output_ctx *out,
+                                          struct runtime_stat *st)
 {
         double total, ratio = 0.0;
         const char *color;
         int ctx = evsel_context(evsel);
 
-        total = avg_stats(&runtime_cycles_stats[ctx][cpu]);
+        total = runtime_stat_avg(st, STAT_CYCLES, ctx, cpu);
 
         if (total)
                 ratio = avg / total * 100.0;
@@ -376,13 +435,14 @@ static void print_stalled_cycles_frontend(int cpu,
 
 static void print_stalled_cycles_backend(int cpu,
                                          struct perf_evsel *evsel, double avg,
-                                         struct perf_stat_output_ctx *out)
+                                         struct perf_stat_output_ctx *out,
+                                         struct runtime_stat *st)
 {
         double total, ratio = 0.0;
         const char *color;
         int ctx = evsel_context(evsel);
 
-        total = avg_stats(&runtime_cycles_stats[ctx][cpu]);
+        total = runtime_stat_avg(st, STAT_CYCLES, ctx, cpu);
 
         if (total)
                 ratio = avg / total * 100.0;
@@ -395,13 +455,14 @@ static void print_stalled_cycles_backend(int cpu,
 static void print_branch_misses(int cpu,
                                 struct perf_evsel *evsel,
                                 double avg,
-                                struct perf_stat_output_ctx *out)
+                                struct perf_stat_output_ctx *out,
+                                struct runtime_stat *st)
 {
         double total, ratio = 0.0;
         const char *color;
         int ctx = evsel_context(evsel);
 
-        total = avg_stats(&runtime_branches_stats[ctx][cpu]);
+        total = runtime_stat_avg(st, STAT_BRANCHES, ctx, cpu);
 
         if (total)
                 ratio = avg / total * 100.0;
@@ -414,13 +475,15 @@ static void print_branch_misses(int cpu,
 static void print_l1_dcache_misses(int cpu,
                                    struct perf_evsel *evsel,
                                    double avg,
-                                   struct perf_stat_output_ctx *out)
+                                   struct perf_stat_output_ctx *out,
+                                   struct runtime_stat *st)
+
 {
         double total, ratio = 0.0;
         const char *color;
         int ctx = evsel_context(evsel);
 
-        total = avg_stats(&runtime_l1_dcache_stats[ctx][cpu]);
+        total = runtime_stat_avg(st, STAT_L1_DCACHE, ctx, cpu);
 
         if (total)
                 ratio = avg / total * 100.0;
@@ -433,13 +496,15 @@ static void print_l1_dcache_misses(int cpu,
 static void print_l1_icache_misses(int cpu,
                                    struct perf_evsel *evsel,
                                    double avg,
-                                   struct perf_stat_output_ctx *out)
+                                   struct perf_stat_output_ctx *out,
+                                   struct runtime_stat *st)
+
 {
         double total, ratio = 0.0;
         const char *color;
         int ctx = evsel_context(evsel);
 
-        total = avg_stats(&runtime_l1_icache_stats[ctx][cpu]);
+        total = runtime_stat_avg(st, STAT_L1_ICACHE, ctx, cpu);
 
         if (total)
                 ratio = avg / total * 100.0;
@@ -451,13 +516,14 @@ static void print_l1_icache_misses(int cpu,
 static void print_dtlb_cache_misses(int cpu,
                                     struct perf_evsel *evsel,
                                     double avg,
-                                    struct perf_stat_output_ctx *out)
+                                    struct perf_stat_output_ctx *out,
+                                    struct runtime_stat *st)
 {
         double total, ratio = 0.0;
         const char *color;
         int ctx = evsel_context(evsel);
 
-        total = avg_stats(&runtime_dtlb_cache_stats[ctx][cpu]);
+        total = runtime_stat_avg(st, STAT_DTLB_CACHE, ctx, cpu);
 
         if (total)
                 ratio = avg / total * 100.0;
@@ -469,13 +535,14 @@ static void print_dtlb_cache_misses(int cpu,
 static void print_itlb_cache_misses(int cpu,
                                     struct perf_evsel *evsel,
                                     double avg,
-                                    struct perf_stat_output_ctx *out)
+                                    struct perf_stat_output_ctx *out,
+                                    struct runtime_stat *st)
 {
         double total, ratio = 0.0;
         const char *color;
         int ctx = evsel_context(evsel);
 
-        total = avg_stats(&runtime_itlb_cache_stats[ctx][cpu]);
+        total = runtime_stat_avg(st, STAT_ITLB_CACHE, ctx, cpu);
 
         if (total)
                 ratio = avg / total * 100.0;
@@ -487,13 +554,14 @@ static void print_itlb_cache_misses(int cpu,
 static void print_ll_cache_misses(int cpu,
                                   struct perf_evsel *evsel,
                                   double avg,
-                                  struct perf_stat_output_ctx *out)
+                                  struct perf_stat_output_ctx *out,
+                                  struct runtime_stat *st)
 {
         double total, ratio = 0.0;
         const char *color;
         int ctx = evsel_context(evsel);
 
-        total = avg_stats(&runtime_ll_cache_stats[ctx][cpu]);
+        total = runtime_stat_avg(st, STAT_LL_CACHE, ctx, cpu);
 
         if (total)
                 ratio = avg / total * 100.0;
@@ -551,68 +619,72 @@ static double sanitize_val(double x)
         return x;
 }
 
-static double td_total_slots(int ctx, int cpu)
+static double td_total_slots(int ctx, int cpu, struct runtime_stat *st)
 {
-        return avg_stats(&runtime_topdown_total_slots[ctx][cpu]);
+        return runtime_stat_avg(st, STAT_TOPDOWN_TOTAL_SLOTS, ctx, cpu);
 }
 
-static double td_bad_spec(int ctx, int cpu)
+static double td_bad_spec(int ctx, int cpu, struct runtime_stat *st)
 {
         double bad_spec = 0;
         double total_slots;
         double total;
 
-        total = avg_stats(&runtime_topdown_slots_issued[ctx][cpu]) -
-                avg_stats(&runtime_topdown_slots_retired[ctx][cpu]) +
-                avg_stats(&runtime_topdown_recovery_bubbles[ctx][cpu]);
-        total_slots = td_total_slots(ctx, cpu);
+        total = runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_ISSUED, ctx, cpu) -
+                runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_RETIRED, ctx, cpu) +
+                runtime_stat_avg(st, STAT_TOPDOWN_RECOVERY_BUBBLES, ctx, cpu);
+
+        total_slots = td_total_slots(ctx, cpu, st);
         if (total_slots)
                 bad_spec = total / total_slots;
         return sanitize_val(bad_spec);
 }
 
-static double td_retiring(int ctx, int cpu)
+static double td_retiring(int ctx, int cpu, struct runtime_stat *st)
 {
         double retiring = 0;
-        double total_slots = td_total_slots(ctx, cpu);
-        double ret_slots = avg_stats(&runtime_topdown_slots_retired[ctx][cpu]);
+        double total_slots = td_total_slots(ctx, cpu, st);
+        double ret_slots = runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_RETIRED,
+                                            ctx, cpu);
 
         if (total_slots)
                 retiring = ret_slots / total_slots;
         return retiring;
 }
 
-static double td_fe_bound(int ctx, int cpu)
+static double td_fe_bound(int ctx, int cpu, struct runtime_stat *st)
 {
         double fe_bound = 0;
-        double total_slots = td_total_slots(ctx, cpu);
-        double fetch_bub = avg_stats(&runtime_topdown_fetch_bubbles[ctx][cpu]);
+        double total_slots = td_total_slots(ctx, cpu, st);
+        double fetch_bub = runtime_stat_avg(st, STAT_TOPDOWN_FETCH_BUBBLES,
+                                            ctx, cpu);
 
         if (total_slots)
                 fe_bound = fetch_bub / total_slots;
         return fe_bound;
 }
 
-static double td_be_bound(int ctx, int cpu)
+static double td_be_bound(int ctx, int cpu, struct runtime_stat *st)
 {
-        double sum = (td_fe_bound(ctx, cpu) +
-                      td_bad_spec(ctx, cpu) +
-                      td_retiring(ctx, cpu));
+        double sum = (td_fe_bound(ctx, cpu, st) +
+                      td_bad_spec(ctx, cpu, st) +
+                      td_retiring(ctx, cpu, st));
         if (sum == 0)
                 return 0;
         return sanitize_val(1.0 - sum);
 }
 
 static void print_smi_cost(int cpu, struct perf_evsel *evsel,
-                           struct perf_stat_output_ctx *out)
+                           struct perf_stat_output_ctx *out,
+                           struct runtime_stat *st)
 {
         double smi_num, aperf, cycles, cost = 0.0;
         int ctx = evsel_context(evsel);
         const char *color = NULL;
 
-        smi_num = avg_stats(&runtime_smi_num_stats[ctx][cpu]);
-        aperf = avg_stats(&runtime_aperf_stats[ctx][cpu]);
-        cycles = avg_stats(&runtime_cycles_stats[ctx][cpu]);
+        smi_num = runtime_stat_avg(st, STAT_SMI_NUM, ctx, cpu);
+        aperf = runtime_stat_avg(st, STAT_APERF, ctx, cpu);
+        cycles = runtime_stat_avg(st, STAT_CYCLES, ctx, cpu);
 
         if ((cycles == 0) || (aperf == 0))
                 return;
@@ -632,7 +704,8 @@ static void generic_metric(const char *metric_expr,
                            const char *metric_name,
                            double avg,
                            int cpu,
-                           struct perf_stat_output_ctx *out)
+                           struct perf_stat_output_ctx *out,
+                           struct runtime_stat *st)
 {
         print_metric_t print_metric = out->print_metric;
         struct parse_ctx pctx;
@@ -651,7 +724,8 @@ static void generic_metric(const char *metric_expr,
                         stats = &walltime_nsecs_stats;
                         scale = 1e-9;
                 } else {
-                        v = saved_value_lookup(metric_events[i], cpu, false);
+                        v = saved_value_lookup(metric_events[i], cpu, false,
+                                               STAT_NONE, 0, st);
                         if (!v)
                                 break;
                         stats = &v->stats;
@@ -679,7 +753,8 @@ static void generic_metric(const char *metric_expr,
 void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                                    double avg, int cpu,
                                    struct perf_stat_output_ctx *out,
-                                   struct rblist *metric_events)
+                                   struct rblist *metric_events,
+                                   struct runtime_stat *st)
 {
         void *ctxp = out->ctx;
         print_metric_t print_metric = out->print_metric;
@@ -690,7 +765,8 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
         int num = 1;
 
         if (perf_evsel__match(evsel, HARDWARE, HW_INSTRUCTIONS)) {
-                total = avg_stats(&runtime_cycles_stats[ctx][cpu]);
+                total = runtime_stat_avg(st, STAT_CYCLES, ctx, cpu);
+
                 if (total) {
                         ratio = avg / total;
                         print_metric(ctxp, NULL, "%7.2f ",
@@ -698,8 +774,13 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                 } else {
                         print_metric(ctxp, NULL, NULL, "insn per cycle", 0);
                 }
-                total = avg_stats(&runtime_stalled_cycles_front_stats[ctx][cpu]);
-                total = max(total, avg_stats(&runtime_stalled_cycles_back_stats[ctx][cpu]));
+
+                total = runtime_stat_avg(st, STAT_STALLED_CYCLES_FRONT,
+                                         ctx, cpu);
+
+                total = max(total, runtime_stat_avg(st,
+                                                    STAT_STALLED_CYCLES_BACK,
+                                                    ctx, cpu));
 
                 if (total && avg) {
                         out->new_line(ctxp);
@@ -712,8 +793,8 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                                      "stalled cycles per insn", 0);
                 }
         } else if (perf_evsel__match(evsel, HARDWARE, HW_BRANCH_MISSES)) {
-                if (runtime_branches_stats[ctx][cpu].n != 0)
-                        print_branch_misses(cpu, evsel, avg, out);
+                if (runtime_stat_n(st, STAT_BRANCHES, ctx, cpu) != 0)
+                        print_branch_misses(cpu, evsel, avg, out, st);
                 else
                         print_metric(ctxp, NULL, NULL, "of all branches", 0);
         } else if (
@@ -721,8 +802,9 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                 evsel->attr.config ==  ( PERF_COUNT_HW_CACHE_L1D |
                                         ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
                                          ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
-                if (runtime_l1_dcache_stats[ctx][cpu].n != 0)
-                        print_l1_dcache_misses(cpu, evsel, avg, out);
+
+                if (runtime_stat_n(st, STAT_L1_DCACHE, ctx, cpu) != 0)
+                        print_l1_dcache_misses(cpu, evsel, avg, out, st);
                 else
                         print_metric(ctxp, NULL, NULL, "of all L1-dcache hits", 0);
         } else if (
@@ -730,8 +812,9 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                 evsel->attr.config ==  ( PERF_COUNT_HW_CACHE_L1I |
                                         ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
                                          ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
-                if (runtime_l1_icache_stats[ctx][cpu].n != 0)
-                        print_l1_icache_misses(cpu, evsel, avg, out);
+
+                if (runtime_stat_n(st, STAT_L1_ICACHE, ctx, cpu) != 0)
+                        print_l1_icache_misses(cpu, evsel, avg, out, st);
                 else
                         print_metric(ctxp, NULL, NULL, "of all L1-icache hits", 0);
         } else if (
@@ -739,8 +822,9 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                 evsel->attr.config ==  ( PERF_COUNT_HW_CACHE_DTLB |
                                         ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
                                          ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
-                if (runtime_dtlb_cache_stats[ctx][cpu].n != 0)
-                        print_dtlb_cache_misses(cpu, evsel, avg, out);
+
+                if (runtime_stat_n(st, STAT_DTLB_CACHE, ctx, cpu) != 0)
+                        print_dtlb_cache_misses(cpu, evsel, avg, out, st);
                 else
                         print_metric(ctxp, NULL, NULL, "of all dTLB cache hits", 0);
         } else if (
@@ -748,8 +832,9 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                 evsel->attr.config ==  ( PERF_COUNT_HW_CACHE_ITLB |
                                         ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
                                          ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
-                if (runtime_itlb_cache_stats[ctx][cpu].n != 0)
-                        print_itlb_cache_misses(cpu, evsel, avg, out);
+
+                if (runtime_stat_n(st, STAT_ITLB_CACHE, ctx, cpu) != 0)
+                        print_itlb_cache_misses(cpu, evsel, avg, out, st);
                 else
                         print_metric(ctxp, NULL, NULL, "of all iTLB cache hits", 0);
         } else if (
@@ -757,27 +842,28 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                 evsel->attr.config ==  ( PERF_COUNT_HW_CACHE_LL |
                                         ((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
                                          ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
-                if (runtime_ll_cache_stats[ctx][cpu].n != 0)
-                        print_ll_cache_misses(cpu, evsel, avg, out);
+
+                if (runtime_stat_n(st, STAT_LL_CACHE, ctx, cpu) != 0)
+                        print_ll_cache_misses(cpu, evsel, avg, out, st);
                 else
                         print_metric(ctxp, NULL, NULL, "of all LL-cache hits", 0);
         } else if (perf_evsel__match(evsel, HARDWARE, HW_CACHE_MISSES)) {
-                total = avg_stats(&runtime_cacherefs_stats[ctx][cpu]);
+                total = runtime_stat_avg(st, STAT_CACHEREFS, ctx, cpu);
 
                 if (total)
                         ratio = avg * 100 / total;
 
-                if (runtime_cacherefs_stats[ctx][cpu].n != 0)
+                if (runtime_stat_n(st, STAT_CACHEREFS, ctx, cpu) != 0)
                         print_metric(ctxp, NULL, "%8.3f %%",
                                      "of all cache refs", ratio);
                 else
                         print_metric(ctxp, NULL, NULL, "of all cache refs", 0);
         } else if (perf_evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_FRONTEND)) {
-                print_stalled_cycles_frontend(cpu, evsel, avg, out);
+                print_stalled_cycles_frontend(cpu, evsel, avg, out, st);
         } else if (perf_evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_BACKEND)) {
-                print_stalled_cycles_backend(cpu, evsel, avg, out);
+                print_stalled_cycles_backend(cpu, evsel, avg, out, st);
         } else if (perf_evsel__match(evsel, HARDWARE, HW_CPU_CYCLES)) {
-                total = avg_stats(&runtime_nsecs_stats[cpu]);
+                total = runtime_stat_avg(st, STAT_NSECS, 0, cpu);
 
                 if (total) {
                         ratio = avg / total;
@@ -786,7 +872,8 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                         print_metric(ctxp, NULL, NULL, "Ghz", 0);
                 }
         } else if (perf_stat_evsel__is(evsel, CYCLES_IN_TX)) {
-                total = avg_stats(&runtime_cycles_stats[ctx][cpu]);
+                total = runtime_stat_avg(st, STAT_CYCLES, ctx, cpu);
+
                 if (total)
                         print_metric(ctxp, NULL,
                                         "%7.2f%%", "transactional cycles",
@@ -795,8 +882,9 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                         print_metric(ctxp, NULL, NULL, "transactional cycles",
                                      0);
         } else if (perf_stat_evsel__is(evsel, CYCLES_IN_TX_CP)) {
-                total = avg_stats(&runtime_cycles_stats[ctx][cpu]);
-                total2 = avg_stats(&runtime_cycles_in_tx_stats[ctx][cpu]);
+                total = runtime_stat_avg(st, STAT_CYCLES, ctx, cpu);
+                total2 = runtime_stat_avg(st, STAT_CYCLES_IN_TX, ctx, cpu);
+
                 if (total2 < avg)
                         total2 = avg;
                 if (total)
@@ -805,19 +893,21 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                 else
                         print_metric(ctxp, NULL, NULL, "aborted cycles", 0);
         } else if (perf_stat_evsel__is(evsel, TRANSACTION_START)) {
-                total = avg_stats(&runtime_cycles_in_tx_stats[ctx][cpu]);
+                total = runtime_stat_avg(st, STAT_CYCLES_IN_TX,
+                                         ctx, cpu);
 
                 if (avg)
                         ratio = total / avg;
 
-                if (runtime_cycles_in_tx_stats[ctx][cpu].n != 0)
+                if (runtime_stat_n(st, STAT_CYCLES_IN_TX, ctx, cpu) != 0)
                         print_metric(ctxp, NULL, "%8.0f",
                                      "cycles / transaction", ratio);
                 else
                         print_metric(ctxp, NULL, NULL, "cycles / transaction",
-                                     0);
+                                      0);
         } else if (perf_stat_evsel__is(evsel, ELISION_START)) {
-                total = avg_stats(&runtime_cycles_in_tx_stats[ctx][cpu]);
+                total = runtime_stat_avg(st, STAT_CYCLES_IN_TX,
+                                         ctx, cpu);
 
                 if (avg)
                         ratio = total / avg;
@@ -831,28 +921,28 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                 else
                         print_metric(ctxp, NULL, NULL, "CPUs utilized", 0);
         } else if (perf_stat_evsel__is(evsel, TOPDOWN_FETCH_BUBBLES)) {
-                double fe_bound = td_fe_bound(ctx, cpu);
+                double fe_bound = td_fe_bound(ctx, cpu, st);
 
                 if (fe_bound > 0.2)
                         color = PERF_COLOR_RED;
                 print_metric(ctxp, color, "%8.1f%%", "frontend bound",
                                 fe_bound * 100.);
         } else if (perf_stat_evsel__is(evsel, TOPDOWN_SLOTS_RETIRED)) {
-                double retiring = td_retiring(ctx, cpu);
+                double retiring = td_retiring(ctx, cpu, st);
 
                 if (retiring > 0.7)
                         color = PERF_COLOR_GREEN;
                 print_metric(ctxp, color, "%8.1f%%", "retiring",
                                 retiring * 100.);
         } else if (perf_stat_evsel__is(evsel, TOPDOWN_RECOVERY_BUBBLES)) {
-                double bad_spec = td_bad_spec(ctx, cpu);
+                double bad_spec = td_bad_spec(ctx, cpu, st);
 
                 if (bad_spec > 0.1)
                         color = PERF_COLOR_RED;
                 print_metric(ctxp, color, "%8.1f%%", "bad speculation",
                                 bad_spec * 100.);
         } else if (perf_stat_evsel__is(evsel, TOPDOWN_SLOTS_ISSUED)) {
-                double be_bound = td_be_bound(ctx, cpu);
+                double be_bound = td_be_bound(ctx, cpu, st);
                 const char *name = "backend bound";
                 static int have_recovery_bubbles = -1;
 
@@ -865,19 +955,19 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
 
                 if (be_bound > 0.2)
                         color = PERF_COLOR_RED;
-                if (td_total_slots(ctx, cpu) > 0)
+                if (td_total_slots(ctx, cpu, st) > 0)
                         print_metric(ctxp, color, "%8.1f%%", name,
                                         be_bound * 100.);
                 else
                         print_metric(ctxp, NULL, NULL, name, 0);
         } else if (evsel->metric_expr) {
                 generic_metric(evsel->metric_expr, evsel->metric_events, evsel->name,
-                                evsel->metric_name, avg, cpu, out);
-        } else if (runtime_nsecs_stats[cpu].n != 0) {
+                                evsel->metric_name, avg, cpu, out, st);
+        } else if (runtime_stat_n(st, STAT_NSECS, 0, cpu) != 0) {
                 char unit = 'M';
                 char unit_buf[10];
 
-                total = avg_stats(&runtime_nsecs_stats[cpu]);
+                total = runtime_stat_avg(st, STAT_NSECS, 0, cpu);
 
                 if (total)
                         ratio = 1000.0 * avg / total;
@@ -888,7 +978,7 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                 snprintf(unit_buf, sizeof(unit_buf), "%c/sec", unit);
                 print_metric(ctxp, NULL, "%8.3f", unit_buf, ratio);
         } else if (perf_stat_evsel__is(evsel, SMI_NUM)) {
-                print_smi_cost(cpu, evsel, out);
+                print_smi_cost(cpu, evsel, out, st);
         } else {
                 num = 0;
         }
@@ -901,7 +991,7 @@ void perf_stat__print_shadow_stats(struct perf_evsel *evsel,
                                 out->new_line(ctxp);
                         generic_metric(mexp->metric_expr, mexp->metric_events,
                                         evsel->name, mexp->metric_name,
-                                        avg, cpu, out);
+                                        avg, cpu, out, st);
                 }
         }
         if (num == 0)