Added script to parse directory data, create CSVs for every chagned value.

This change also makes SchedTrace and OverheadTrace events configurable.

7 files changed, 471 insertions, 0 deletions

diff --git a/parse/__init__.py b/parse/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/parse/__init__.py
diff --git a/parse/dir_map.py b/parse/dir_map.py
new file mode 100644
index 0000000..6e959f2
--- /dev/null
+++ b/parse/dir_map.py
@@ -0,0 +1,104 @@
+import os
+
+from collections import defaultdict
+from point import Type
+
+class TreeNode(object):
+    def __init__(self, parent = None):
+        self.parent = parent
+        self.children = defaultdict(lambda : TreeNode(self))
+        self.values = []
+
+class DirMap(object):
+    def to_csv(self, vals):
+        val_strs = []
+        for key in sorted(vals.keys()):
+            val_strs += ["%s=%s" % (key, vals[key])]
+        return "%s.csv" % ("_".join(val_strs))
+    
+    def __init__(self, out_dir):
+        self.root = TreeNode(None)
+        self.out_dir = out_dir
+        self.values  = []
+
+    def debug_update_node(self, path, keys, value):
+        self.__update_node(path, keys, value)
+
+    def __update_node(self, path, keys, value):
+        node = self.root
+
+        path += [ self.to_csv(keys) ]
+        for p in path:
+            node = node.children[p]
+
+        node.values += [value]
+
+    def add_point(self, vary, vary_value, keys, point):
+        for stat in point.get_stats():
+            summary = point[stat]
+
+            for summary_type in Type:
+                measurement = summary[summary_type]
+
+                for base_type in Type:
+                    if not base_type in measurement:
+                        continue
+                    # Ex: wcet/avg/max/vary-type/other-stuff.csv
+                    path  = [ stat, summary_type, base_type, "vary-%s" % vary ]
+                    result = measurement[base_type]
+
+                    self.__update_node(path, keys, (vary_value, result))
+
+
+                        
+    def reduce(self):
+        def reduce2(node):
+            for key in node.children.keys():
+                child = node.children[key]
+                reduce2(child)
+                if not (child.children or child.values):
+                    node.children.pop(key)
+
+            if len(node.values) == 1:
+                node.values = []
+
+        reduce2(self.root)
+
+    def write(self):    
+        def write2(path, node):
+            out_path = "/".join(path)
+            if node.values:
+                # Leaf
+                with open("/".join(path), "w") as f:
+                    arr = [",".join([str(b) for b in n]) for n in node.values]
+                    f.write("\n".join(arr) + "\n")
+            elif not os.path.isdir(out_path):
+                os.mkdir(out_path)        
+
+            for (key, child) in node.children.iteritems():
+                path.append(key)
+                write2(path, child)
+                path.pop()
+
+
+        write2([self.out_dir], self.root)
+
+
+    def __str__(self):
+        def str2(node, level):
+            header = "  " * level
+            ret = ""
+            if not node.children:
+                return "%s%s\n" % (header, str(node.values) if node.values else "")
+            for key,child in node.children.iteritems():
+                ret += "%s/%s\n" % (header, key)
+                ret += str2(child, level + 1)
+            return ret
+            
+        return "%s\n%s" % (self.out_dir, str2(self.root, 1))
+       
+        
+                
+        
+                
+            
diff --git a/parse/enum.py b/parse/enum.py
new file mode 100644
index 0000000..bf35d01
--- /dev/null
+++ b/parse/enum.py
@@ -0,0 +1,7 @@
+class Enum(frozenset):
+    def __getattr__(self, name):
+        if name in self:
+            return name
+        raise AttributeError
+
+    
diff --git a/parse/ft.py b/parse/ft.py
new file mode 100644
index 0000000..9837898
--- /dev/null
+++ b/parse/ft.py
@@ -0,0 +1,60 @@
+import config.config as conf
+import os
+import re
+import shutil as sh
+import subprocess
+
+from point import Measurement,Type
+
+def get_ft_output(data_dir, out_dir):
+    bin_file = conf.FILES['ft_data'] + "$"
+    bins = [f for f in os.listdir(data_dir) if re.match(bin_file, f)]
+
+    FT_DATA_NAME = "scheduler=x-ft"
+    output_file  = "{}/out-ft".format(out_dir)
+    
+    if os.path.isfile(output_file):
+        print("ft-output already exists for %s" % data_dir)
+        return output_file
+
+    if len(bins) != 0:
+        err_file = open("%s/err-ft" % out_dir, 'w')
+        # Need to make a copy of the original data file so scripts can change it
+        sh.copyfile("{}/{}".format(data_dir, bins[0]),
+                    "{}/{}".format(out_dir, FT_DATA_NAME))
+
+        subprocess.call([conf.BINS['sort'], FT_DATA_NAME],
+                        cwd=out_dir, stderr=err_file, stdout=err_file)
+        subprocess.call([conf.BINS['split'], FT_DATA_NAME],
+                        cwd=out_dir, stderr=err_file, stdout=err_file)
+
+        # Previous subprocesses just spit out all these intermediate files
+        bins = [f for f in os.listdir(out_dir) if re.match(".*overhead=.*bin", f)]
+        bins = [f for f in bins if os.stat("%s/%s"%(out_dir, f)).st_size]
+
+        # Analyze will summarize those
+        cmd_arr = [conf.BINS['analyze']]
+        cmd_arr.extend(bins)
+        with open(output_file, "w") as f:
+            subprocess.call(cmd_arr, cwd=out_dir, stdout=f, stderr=err_file)
+    else:
+        return None
+    return output_file
+
+def get_ft_data(data_file, result, overheads):
+    rstr = r",(?:\s+[^\s]+){3}.*?([\d\.]+).*?([\d\.]+),(?:\s+[^\s]+){3}.*?([\d\.]+)"
+
+    with open(data_file) as f:
+        data = f.read()
+
+    for ovh in overheads:
+        measure = Measurement("%s-%s" % (data_file, ovh))
+        vals = re.findall(".*{}".format(ovh) + rstr, data);
+        if len(vals) != 0:
+            vals = vals[0]
+            measure[Type.Max] = float(vals[0])
+            measure[Type.Avg] = float(vals[1])
+            measure[Type.Var] = float(vals[2])
+            result[ovh] = measure
+
+    return result
diff --git a/parse/point.py b/parse/point.py
new file mode 100644
index 0000000..4343d03
--- /dev/null
+++ b/parse/point.py
@@ -0,0 +1,135 @@
+"""
+Too much duplicate code in this file
+"""
+
+import copy
+import numpy as np
+from enum import Enum
+from collections import defaultdict
+
+Type = Enum(['Min','Max','Avg','Var'])
+default_typemap = {Type.Max : {Type.Max : 1, Type.Min : 0, Type.Avg : 1, Type.Var : 1},
+                   Type.Min : {Type.Max : 1, Type.Min : 0, Type.Avg : 1, Type.Var : 1},
+                   Type.Avg : {Type.Max : 1, Type.Min : 0, Type.Avg : 1, Type.Var : 1}}
+
+def make_typemap():
+    return copy.deepcopy(default_typemap)
+
+def dict_str(adict, sep = "\n"):
+    return sep.join(["%s: %s" % (k, str(v)) for (k,v) in adict.iteritems()])
+
+class Measurement(object):
+    def __init__(self, id = None, kv = {}):
+        self.id = id
+        self.stats = {}
+        for k, v in kv.iteritems():
+            self[k] = v
+
+    def from_array(self,array):
+        array = np.array(array)
+        self[Type.Max] = array.max()
+        self[Type.Avg] = array.mean()
+        self[Type.Var] = array.var()
+        return self
+
+    def __check_type(self, type):
+        if not type in Type: