Prettied up parse output.

6 files changed, 182 insertions, 26 deletions

diff --git a/config/config.example.py b/config/config.example.py
index 06f06b6..50d30ba 100644
--- a/config/config.example.py
+++ b/config/config.example.py
@@ -41,16 +41,22 @@ PARAMS = {'sched'  : 'scheduler',
           'kernel' : 'uname'}
 
 SCHED_EVENTS = range(501, 513)
-BASE_EVENTS  = ['SCHED', 'RELEASE', 'SCHED2', 'TICK', 'CXS']
+BASE_EVENTS  = ['SCHED', 'RELEASE', 'SCHED2', 'TICK', 'CXS', 'SEND_RESCHED']
+BASE_EVENTS += ['CQ_ENQUEUE_READ', 'CQ_ENQUEUE_FLUSH', 'CQ_SUBMIT_WORK',
+                'CQ_LOOP_WORK_CHECK', 'CQ_LOOP_PEACE_OUT', 'CQ_LOOP_BRANCH',
+                'CQ_WORK_DO_WORK', 'CQ_WORK_NOTIFY', 'CQ_PHASE_WAIT']
 
 # Expand for mixed-crit
-# CRIT_EVENTS  = ['LVL{}_SCHED', 'LVL{}_RELEASE']
-# CRIT_LEVELS  = ['A', 'B', 'C']
-# BASE_EVENTS += [s.format(l) for (l,s) in
-#                 itertools.product(CRIT_LEVELS, CRIT_EVENTS)]
+# TODO don't use split
+CRIT_EVENTS  = ['LVL{}_SCHED', 'LVL{}_RELEASE']
+CRIT_LEVELS  = ['A', 'B', 'C']
+BASE_EVENTS += [s.format(l) for (l,s) in
+                itertools.product(CRIT_LEVELS, CRIT_EVENTS)]
 
 ALL_EVENTS   = ["%s_%s" % (e, t) for (e,t) in
                 itertools.product(BASE_EVENTS, ["START","END"])]
+ALL_EVENTS  += ['RELEASE_LATENCY']
+BASE_EVENTS += ['RELEASE_LATENCY']
 
 valid = True
 for repo, loc in REPOS.items():
diff --git a/parse/ft.py b/parse/ft.py
index 2c2b597..feb338f 100644
--- a/parse/ft.py
+++ b/parse/ft.py
@@ -41,6 +41,7 @@ def get_ft_output(data_dir, out_dir, force=False):
         # Analyze will summarize those
         # todo pass in f
         cmd_arr = [conf.BINS['analyze']]
+        print("cmd arr: %s-%s" % (cmd_arr, bins))
         cmd_arr.extend(bins)
         with open(output_file, "w") as f:
             subprocess.call(cmd_arr, cwd=out_dir, stdout=f, stderr=err_file)
diff --git a/parse/point.py b/parse/point.py
index 30fcd97..8fdd115 100644
--- a/parse/point.py
+++ b/parse/point.py
@@ -16,7 +16,14 @@ def make_typemap():
     return copy.deepcopy(default_typemap)
 
 def dict_str(adict, sep = "\n"):
-    return sep.join(["%s: %s" % (k, str(v)) for (k,v) in sorted(adict.iteritems())])
+    def num_str(v):
+        try:
+            float(v)
+            return "%6.3f" % v
+        except:
+            return v
+    size = 20 if sep == "\n" else 4
+    return sep.join([("%" + str(size) + "s: %9s") % (k, num_str(v)) for (k,v) in sorted(adict.iteritems())])
 
 class Measurement(object):
     def __init__(self, id = None, kv = {}):
@@ -52,7 +59,7 @@ class Measurement(object):
         self.stats[type] = value
 
     def __str__(self):
-        return "<Measurement-%s> %s" % (self.id, dict_str(self.stats, " "))
+        return "%s" % dict_str(self.stats, " ")
             
 
 class Summary(Measurement):
diff --git a/parse/sched.py b/parse/sched.py
index 7dd80e0..cbb051e 100644
--- a/parse/sched.py
+++ b/parse/sched.py
@@ -1,5 +1,6 @@
 """
 TODO: No longer very pythonic, lot of duplicate code
+print out task execution times
 """
 
 import config.config as conf
@@ -9,6 +10,7 @@ import numpy as np
 import subprocess
 
 from collections import namedtuple,defaultdict
+from operator import methodcaller
 from point import Measurement,Type
 
 PARAM_RECORD = r"(?P<RECORD>" +\
@@ -29,12 +31,14 @@ TARDY_RECORD = r"(?P<RECORD>" +\
   r"(?P<MISSES>[\d\.]+))"
 COMPLETION_RECORD = r"(?P<RECORD>" +\
   r"COMPLETION.*?(?P<PID>\d+)/.*?" +\
-  r"exec.*?(?P<EXEC>[\d\.]+)ms.*?"    +\
-  r"flush.*?(?P<FLUSH>[\d\.]+)ms.*?"  +\
-  r"load.*?(?P<LOAD>[\d\.]+)ms)"
+  r"exec:.*?(?P<EXEC>[\d\.]+)ms.*?"    +\
+  r"flush:.*?(?P<FLUSH>[\d\.]+)ms.*?"  +\
+  r"flush_work:.*?(?P<FLUSH_WORK>[\d]+).*?" +\
+  r"load:.*?(?P<LOAD>[\d\.]+)ms.*?"    +\
+  r"load_work:.*?(?P<LOAD_WORK>[\d]+))"
 
 TaskConfig = namedtuple('TaskConfig', ['cpu','wcet','period','type','level'])
-Task = namedtuple('Task', ['pid', 'config'])
+Task = namedtuple('Task', ['pid', 'config', 'run'])
 
 class LeveledArray(object):
     """
@@ -86,7 +90,7 @@ def get_tasks(data):
                                   float(match.group('WCET')),
                                   float(match.group('PERIOD')),
                                   match.group("CLASS"),
-                                  match.group("LEVEL")))
+                                  match.group("LEVEL")), [])
             if not (t.config.period and t.pid):
                 raise Exception()
             ret += [t]
@@ -144,15 +148,16 @@ def extract_tardy_vals(task_dict, data, exp_point):
         max_tards.add(t, max_tard / t.config.period)
         ratios.add(t, misses / jobs)
 
-    ratios.write_measurements(exp_point)
-    avg_tards.write_measurements(exp_point)
-    max_tards.write_measurements(exp_point)
+    map(methodcaller('write_measurements', exp_point),
+        [ratios, avg_tards, max_tards])
 
 # TODO: rename
 def extract_variance(task_dict, data, exp_point):
     varz    = LeveledArray("exec-variance")
     flushes = LeveledArray("cache-flush")
     loads   = LeveledArray("cache-load")
+    fworks  = LeveledArray("flush-work")
+    lworks  = LeveledArray("load-work")
 
     completions = defaultdict(lambda: [])
     missed = defaultdict(lambda: int())
@@ -163,11 +168,17 @@ def extract_variance(task_dict, data, exp_point):
             duration = float(match.group("EXEC"))
             load  = float(match.group("LOAD"))
             flush = float(match.group("FLUSH"))
+            lwork = int(match.group("LOAD_WORK"))
+            fwork = int(match.group("FLUSH_WORK"))
 
             if load:
                 loads.add(task_dict[pid], load)
+                lworks.add(task_dict[pid], lwork)
+                if not lwork: raise Exception()
             if flush:
                 flushes.add(task_dict[pid], flush)
+                fworks.add(task_dict[pid], fwork)
+                if not fwork: raise Exception()
 
             # Last (exit) record often has exec time of 0
             missed[pid] += not bool(duration)
@@ -181,6 +192,9 @@ def extract_variance(task_dict, data, exp_point):
         completions[pid] += [duration]
 
     for pid, durations in completions.iteritems():
+        # TODO: not this, please
+        task_dict[pid].run.append(Measurement(pid).from_array(durations))
+
         job_times = np.array(durations)
         mean = job_times.mean()
 
@@ -194,14 +208,15 @@ def extract_variance(task_dict, data, exp_point):
 
         varz.add(task_dict[pid], corrected)
 
-    varz.write_measurements(exp_point)
-    flushes.write_measurements(exp_point)
-    loads.write_measurements(exp_point)
+    if exp_point:
+        map(methodcaller('write_measurements', exp_point),
+            [varz, flushes, loads, fworks, lworks])
 
 def config_exit_stats(task_dict, data):
-    # Dictionary of task exit measurements by pid
-    exits = get_task_exits(data)
-    exit_dict = dict((e.id, e) for e in exits)
+    # # Dictionary of task exit measurements by pid
+    # exits = get_task_exits(data)
+    # exit_dict = dict((e.id, e) for e in exits)
+    extract_variance(task_dict, data, None)
 
     # Dictionary where keys are configurations, values are list
     # of tasks with those configuratino
@@ -212,11 +227,12 @@ def config_exit_stats(task_dict, data):
     for config in config_dict:
         task_list = sorted(config_dict[config])
 
-        # Replace tasks with corresponding exit stats
-        if not t.pid in exit_dict:
-            raise Exception("Missing exit record for task '%s' in '%s'" %
-                            (t, file.name))
-        exit_list = [exit_dict[t.pid] for t in task_list]
+        # # Replace tasks with corresponding exit stats
+        # if not t.pid in exit_dict:
+        #     raise Exception("Missing exit record for task '%s' in '%s'" %
+        #                     (t, file.name))
+        # exit_list = [exit_dict[t.pid] for t in task_list]
+        exit_list = [t.run[0] for t in task_list]
         config_dict[config] = exit_list
 
     return config_dict
@@ -228,6 +244,7 @@ def get_base_stats(base_file):
     with open(base_file, 'r') as f:
         data = f.read()
     task_dict = get_task_dict(data)
+
     result = config_exit_stats(task_dict, data)
     saved_stats[base_file] = result
     return result
@@ -248,16 +265,21 @@ def extract_scaling_data(task_dict, data, result, base_file):
             # Quit, we are missing a record and can't guarantee
             # a task-to-task comparison
             continue
+
         for data_stat, base_stat in zip(data_stats[config],base_stats[config]):
             if not base_stat[Type.Avg] or not base_stat[Type.Max] or \
                not data_stat[Type.Avg] or not data_stat[Type.Max]:
+               print("missing a thing: {},{}".format(base_stat, data_stat))
                continue
             # How much larger is their exec stat than ours?
+            print("%s vs %s" % (base_stat, data_stat))
             avg_scale = float(base_stat[Type.Avg]) / float(data_stat[Type.Avg])
             max_scale = float(base_stat[Type.Max]) / float(data_stat[Type.Max])
 
             task = task_dict[data_stat.id]
 
+            print("scaling for %s" % data_stat.id)
+
             avg_scales.add(task, avg_scale)
             max_scales.add(task, max_scale)
 
diff --git a/parse/tuple_table.py b/parse/tuple_table.py
index cb5a72a..434eb22 100644
--- a/parse/tuple_table.py
+++ b/parse/tuple_table.py
@@ -48,6 +48,7 @@ class TupleTable(object):
         self.table = defaultdict(lambda: [])
         self.reduced = False
 
+    # TODO: rename, make exp agnostic, extend for exps
     def add_exp(self, kv, point):
         key = self.col_map.get_key(kv)
         self.table[key] += [point]
@@ -56,6 +57,10 @@ class TupleTable(object):
         key = self.col_map.get_key(kv)
         return self.table[key]
 
+    def __contains__(self, kv):
+        key = self.col_map.get_key(kv)
+        return key in self.table
+
     def reduce(self):
         if self.reduced:
             raise Exception("cannot reduce twice!")
diff --git a/plot_exps.py b/plot_exps.py
index 06f43b0..46784bc 100644..100755
--- a/plot_exps.py
+++ b/plot_exps.py
@@ -1,7 +1,77 @@
 #!/usr/bin/env python
 from __future__ import print_function
 
+import os
+import re
+import plot
+import shutil as sh
+
+from collections import defaultdict
 from optparse import OptionParser
+from gnuplot import Plot, curve
+from random import randrange
+
+class StyleMaker(object):
+    LINE_WIDTH = 1.5
+    POINT_SIZE = 0.6
+    BEST_COLORS = [
+            '#ff0000', # red
+            '#000001', # black
+            '#0000ff', # blue
+            '#be00c4', # purple
+            '#ffd700', # yellow
+            ]
+
+    def __init__(csvs):
+        self.main_key, self.col_map = __find_columns(csvs)
+        self.cur_style = 1
+
+        # Use this for least-common varying attribute
+        self.main_map = {}
+        # Everything else is a color
+        self.color_map = TupleTable(self.col_map)
+
+    def __find_columns(csvs):
+        vals = defaultdict(lambda:set)
+
+        for csv in csvs:
+            to_decode = os.path.splitext(csv_file)[0]
+            params = plot.decode(to_decode)
+            for k,v in params.iteritems:
+                vals[k].add(v)
+
+        try:
+            main_key = min([(k,v) for (k,v) in thing.iteritems() if len(v) > 1],
+                key=operator.itemgetter(1))[0]
+        except ValueError:
+            main_key = None
+
+        col_map = ColMap()
+        for k,v in vals.iterkeys():
+            if k == self.main_key: continue
+            for i in v:
+                self.col_map.try_add(k, i)
+        return (main_key, col_map)
+
+    def __rand_color():
+        return "#%s" % "".join([hex(randrange(0, 255))[2:] for i in range(3)])
+
+    def get_style(csv):
+        to_decode = os.path.splitext(csv_file)[0]
+        params = plot.decode(to_decode)
+
+        if kv not in self.color_map:
+            color = best.pop() if BEST_COLORS else __rand_color()
+            self.color_map.add_exp(params, color)
+
+        if self.main_key in params:
+            val = params[self.main_key]
+            if val not in self.main_map:
+                self.main_map[val] = self.cur_style
+                self.cur_style += 1
+            style = self.main_map[val]
+        else:
+            style = 1
 
 def parse_args():
     parser = OptionParser("usage: %prog [options] [csv_dir]...")
@@ -13,9 +83,54 @@ def parse_args():
 
     return parser.parse_args()
 
+def get_label(kv):
+    label = []
+    for key, value in kv.iteritems():
+        label += ["%s=%s" % (key.capitalize(), value)]
+    return ", ".join(label)
+
+def add_line(plot, csv_file):
+    to_decode = os.path.splitext(csv_file)[0]
+    params = plot.decode(to_decode)
+
+def get_stat(path, name):
+    full  = os.path.abspath(path)
+    rstr  = r"(?P<STAT>[^/]+)/((max|min|var|avg)/)*(%s/?)?$" % name
+    regex = re.compile(rstr, re.I | re.M)
+    match = regex.search(full)
+    return match.group("STAT")
+
+def plot_exp(name, data_dir, out_dir):
+    p = Plot()
+    p.format = 'pdf'
+    p.output = "%s/%s.pdf" % (out_dir, name)
+    p.xlabel = name.replace("vary-", "")
+    p.ylabel = get_stat(data_dir, name)
+    p.font   = 'Helvetica'
+    p.dashed_lines  = True
+    p.enhanced_text = True
+    p.size          = ('5.0cm', '5.0cm')
+    p.font_size     = '6pt'
+    p.key           = 'on bmargin center horizontal'
+
+    csvs = [f for f in os.listdir(data_dir) if re.match("*.csv", f)]
+    col_map = get_col_map(csvs)
+
+
 def main():
     opts, args = parse_args()
     args = args or [os.getcwd()]
 
+    # if opts.force and os.path.exists(opts.out_dir):
+    #     sh.rmtree(opts.out_dir)
+    # if not os.path.exists(opts.out_dir):
+    #     os.mkdir(opts.out_dir)
+
+    for exp in args:
+        name = os.path.split(exp)[1]
+        out_dir = "%s/%s" % (opts.out_dir, exp)
+
+        plot_exp(name, exp, out_dir)
+
 if __name__ == '__main__':
     main()