Added parsing for load / flush operations in MC.

5 files changed, 65 insertions, 49 deletions

diff --git a/parse/dir_map.py b/parse/dir_map.py
index 6e959f2..319a5de 100644
--- a/parse/dir_map.py
+++ b/parse/dir_map.py
@@ -15,7 +15,7 @@ class DirMap(object):
         for key in sorted(vals.keys()):
             val_strs += ["%s=%s" % (key, vals[key])]
         return "%s.csv" % ("_".join(val_strs))
-    
+
     def __init__(self, out_dir):
         self.root = TreeNode(None)
         self.out_dir = out_dir
@@ -49,8 +49,6 @@ class DirMap(object):
 
                     self.__update_node(path, keys, (vary_value, result))
 
-
-                        
     def reduce(self):
         def reduce2(node):
             for key in node.children.keys():
@@ -64,7 +62,7 @@ class DirMap(object):
 
         reduce2(self.root)
 
-    def write(self):    
+    def write(self):
         def write2(path, node):
             out_path = "/".join(path)
             if node.values:
@@ -73,7 +71,7 @@ class DirMap(object):
                     arr = [",".join([str(b) for b in n]) for n in node.values]
                     f.write("\n".join(arr) + "\n")
             elif not os.path.isdir(out_path):
-                os.mkdir(out_path)        
+                os.mkdir(out_path)
 
             for (key, child) in node.children.iteritems():
                 path.append(key)
@@ -94,11 +92,4 @@ class DirMap(object):
                 ret += "%s/%s\n" % (header, key)
                 ret += str2(child, level + 1)
             return ret
-            
         return "%s\n%s" % (self.out_dir, str2(self.root, 1))
-       
-        
-                
-        
-                
-            
diff --git a/parse/ft.py b/parse/ft.py
index c915978..2c2b597 100644
--- a/parse/ft.py
+++ b/parse/ft.py
@@ -39,6 +39,7 @@ def get_ft_output(data_dir, out_dir, force=False):
         bins = [f for f in bins if os.stat("%s/%s"%(out_dir, f)).st_size]
 
         # Analyze will summarize those
+        # todo pass in f
         cmd_arr = [conf.BINS['analyze']]
         cmd_arr.extend(bins)
         with open(output_file, "w") as f:
diff --git a/parse/sched.py b/parse/sched.py
index a65f001..7dd80e0 100644
--- a/parse/sched.py
+++ b/parse/sched.py
@@ -13,7 +13,7 @@ from point import Measurement,Type
 
 PARAM_RECORD = r"(?P<RECORD>" +\
   r"PARAM *?(?P<PID>\d+)\/.*?" +\
-  r"cost:\s+(?P<WCET>[\d\.]+)ms.*?" +\
+  r"cost.*?(?P<WCET>[\d\.]+)ms.*?" +\
   r"period.*?(?P<PERIOD>[\d.]+)ms.*?" +\
   r"part.*?(?P<CPU>\d+)[, ]*" +\
   r"(?:class=(?P<CLASS>\w+))?[, ]*" +\
@@ -23,13 +23,15 @@ EXIT_RECORD = r"(?P<RECORD>" +\
   r"Avg.*?(?P<AVG>\d+).*?" +\
   r"Max.*?(?P<MAX>\d+))"
 TARDY_RECORD = r"(?P<RECORD>" +\
-  r"TARDY.*?(?P<PID>\d+)/(?P<JOB>\d+).*?" +\
+  r"TASK_TARDY.*?(?P<PID>\d+)/(?P<JOB>\d+).*?" +\
   r"Tot.*?(?P<TOTAL>[\d\.]+).*?ms.*?" +\
   r"(?P<MAX>[\d\.]+).*?ms.*?" +\
   r"(?P<MISSES>[\d\.]+))"
 COMPLETION_RECORD = r"(?P<RECORD>" +\
   r"COMPLETION.*?(?P<PID>\d+)/.*?" +\
-  r"(?P<EXEC>[\d\.]+)ms)"
+  r"exec.*?(?P<EXEC>[\d\.]+)ms.*?"    +\
+  r"flush.*?(?P<FLUSH>[\d\.]+)ms.*?"  +\
+  r"load.*?(?P<LOAD>[\d\.]+)ms)"
 
 TaskConfig = namedtuple('TaskConfig', ['cpu','wcet','period','type','level'])
 Task = namedtuple('Task', ['pid', 'config'])
@@ -107,11 +109,9 @@ def get_task_exits(data):
             m = Measurement( int(match.group('PID')),
                              {Type.Max : float(match.group('MAX')),
                               Type.Avg : float(match.group('AVG'))})
-            for (type, value) in m:
-                if not value: raise Exception()
         except:
                 raise Exception("Invalid exit record, parsed:\n\t%s\n\t%s" %
-                                (match.groupdict(), m.group('RECORD')))
+                                (match.groupdict(), match.group('RECORD')))
 
         ret += [m]
     return ret
@@ -137,7 +137,7 @@ def extract_tardy_vals(task_dict, data, exp_point):
 
         if pid not in task_dict:
             raise Exception("Invalid pid '%d' in tardy record:\n\t%s" %
-                            match.group("RECORD"))
+                            (pid, match.group("RECORD")))
 
         t = task_dict[pid]
         avg_tards.add(t, total_tard / (jobs * t.config.period))
@@ -148,8 +148,12 @@ def extract_tardy_vals(task_dict, data, exp_point):
     avg_tards.write_measurements(exp_point)
     max_tards.write_measurements(exp_point)
 
+# TODO: rename
 def extract_variance(task_dict, data, exp_point):
-    varz = LeveledArray("exec-variance")
+    varz    = LeveledArray("exec-variance")
+    flushes = LeveledArray("cache-flush")
+    loads   = LeveledArray("cache-load")
+
     completions = defaultdict(lambda: [])
     missed = defaultdict(lambda: int())
 
@@ -157,19 +161,31 @@ def extract_variance(task_dict, data, exp_point):
         try:
             pid = int(match.group("PID"))
             duration = float(match.group("EXEC"))
+            load  = float(match.group("LOAD"))
+            flush = float(match.group("FLUSH"))
+
+            if load:
+                loads.add(task_dict[pid], load)
+            if flush:
+                flushes.add(task_dict[pid], flush)
 
             # Last (exit) record often has exec time of 0
             missed[pid] += not bool(duration)
 
-            if missed[pid] > 1 or not pid: raise Exception()
+            if missed[pid] > 1 or not pid: #TODO: fix, raise Exception()
+                continue
         except:
-            raise Exception("Invalid completion record, missed - %d:"
+            raise Exception("Invalid completion record, missed: %d:"
                             "\n\t%s\n\t%s" % (missed[pid], match.groupdict(),
                                               match.group("RECORD")))
         completions[pid] += [duration]
 
     for pid, durations in completions.iteritems():
         job_times = np.array(durations)
+        mean = job_times.mean()
+
+        if not mean or not durations:
+            continue
 
         # Coefficient of variation
         cv = job_times.std() / job_times.mean()
@@ -179,11 +195,10 @@ def extract_variance(task_dict, data, exp_point):
         varz.add(task_dict[pid], corrected)
 
     varz.write_measurements(exp_point)
+    flushes.write_measurements(exp_point)
+    loads.write_measurements(exp_point)
 
-def config_exit_stats(task_dict, file):
-    with open(file, 'r') as f:
-        data = f.read()
-
+def config_exit_stats(task_dict, data):
     # Dictionary of task exit measurements by pid
     exits = get_task_exits(data)
     exit_dict = dict((e.id, e) for e in exits)
@@ -200,7 +215,7 @@ def config_exit_stats(task_dict, file):
         # Replace tasks with corresponding exit stats
         if not t.pid in exit_dict:
             raise Exception("Missing exit record for task '%s' in '%s'" %
-                            (t, file))
+                            (t, file.name))
         exit_list = [exit_dict[t.pid] for t in task_list]
         config_dict[config] = exit_list
 
@@ -212,13 +227,14 @@ def get_base_stats(base_file):
         return saved_stats[base_file]
     with open(base_file, 'r') as f:
         data = f.read()
-    result = config_exit_stats(data)
+    task_dict = get_task_dict(data)
+    result = config_exit_stats(task_dict, data)
     saved_stats[base_file] = result
     return result
 
 def extract_scaling_data(task_dict, data, result, base_file):
     # Generate trees of tasks with matching configurations
-    data_stats = config_exit_stats(data)
+    data_stats = config_exit_stats(task_dict, data)
     base_stats = get_base_stats(base_file)
 
     # Scaling factors are calculated by matching groups of tasks with the same
@@ -233,9 +249,12 @@ def extract_scaling_data(task_dict, data, result, base_file):
             # a task-to-task comparison
             continue
         for data_stat, base_stat in zip(data_stats[config],base_stats[config]):
+            if not base_stat[Type.Avg] or not base_stat[Type.Max] or \
+               not data_stat[Type.Avg] or not data_stat[Type.Max]:
+               continue
             # How much larger is their exec stat than ours?
-            avg_scale = float(base_stat[Type.Avg]) / float(base_stat[Type.Avg])
-            max_scale = float(base_stat[Type.Max]) / float(base_stat[Type.Max])
+            avg_scale = float(base_stat[Type.Avg]) / float(data_stat[Type.Avg])
+            max_scale = float(base_stat[Type.Max]) / float(data_stat[Type.Max])
 
             task = task_dict[data_stat.id]
 
@@ -251,8 +270,12 @@ def extract_sched_data(data_file, result, base_file):
 
     task_dict = get_task_dict(data)
 
-    extract_tardy_vals(task_dict, data, result)
-    extract_variance(task_dict, data, result)
+    try:
+        extract_tardy_vals(task_dict, data, result)
+        extract_variance(task_dict, data, result)
+    except Exception as e:
+        print("Error in %s" % data_file)
+        raise e
 
     if (base_file):
         extract_scaling_data(task_dict, data, result, base_file)
diff --git a/parse/tuple_table.py b/parse/tuple_table.py
index 5e98d87..cb5a72a 100644
--- a/parse/tuple_table.py
+++ b/parse/tuple_table.py
@@ -5,6 +5,7 @@ from dir_map import DirMap
 class ColMap(object):
     def __init__(self):
         self.rev_map = {}
+        self.value_map = {}
         self.col_list = []
 
     def columns(self):
@@ -12,19 +13,13 @@ class ColMap(object):
 
     def get_key(self, kv):
         key = ()
-        added = 0
 
         for col in self.col_list:
             if col not in kv:
                 key += (None,)
             else:
-                added += 1
                 key += (kv[col],)
 
-        if added < len(kv):
-            raise Exception("column map '%s' missed field in map '%s'" %
-                            (self.col_list, kv))
-
         return key
 
     def __contains__(self, col):
@@ -36,10 +31,13 @@ class ColMap(object):
             map[self.col_list[i]] = tuple[i]
         return map
 
-    def try_add(self, column):
+    def try_add(self, column, value):
         if column not in self.rev_map:
-            self.rev_map[column] = len(self.col_list)
-            self.col_list += [column]
+            if column not in self.value_map:
+                self.value_map[column] = value
+            elif value != self.value_map[column]:
+                self.rev_map[column] = len(self.col_list)
+                self.col_list += [column]
 
     def __str__(self):
         return "<ColMap>%s" % (self.rev_map)
@@ -67,6 +65,7 @@ class TupleTable(object):
 
     def write_result(self, out_dir):
         dir_map = DirMap(out_dir)
+
         for key, point in self.table.iteritems():
             kv = self.col_map.get_map(key)
 
@@ -75,12 +74,13 @@ class TupleTable(object):
 
                 try:
                     float(val)
-                    kv.pop(col)
-                    dir_map.add_point(col, val, kv, point)
-                    kv[col] = val
                 except:
                     # Only vary numbers. Otherwise, just have seperate lines
                     continue
 
+                kv.pop(col)
+                dir_map.add_point(col, val, kv, point)
+                kv[col] = val
+
         dir_map.reduce()
         dir_map.write()
diff --git a/parse_exps.py b/parse_exps.py
index 1f36bab..aa203d3 100755
--- a/parse_exps.py
+++ b/parse_exps.py
@@ -20,6 +20,7 @@ def parse_args():
 
     parser.add_option('-o', '--out-dir', dest='out_dir',
                       help='directory for data output', default='parse-data')
+    # TODO: this means nothing
     parser.add_option('-c', '--clean', action='store_true', default=False,
                       dest='clean', help='do not output single-point csvs')
     parser.add_option('-s', '--scale-against', dest='scale_against',
@@ -47,8 +48,8 @@ def get_exp_params(data_dir, col_map):
             params.pop(ignored)
 
     # Track all changed params
-    for key in params.keys():
-        col_map.try_add(key)
+    for key, value in params.iteritems():
+        col_map.try_add(key, value)
 
     return params
 
@@ -122,6 +123,7 @@ def main():
                 base_params = copy.deepcopy(exp.params)
                 base_params.pop(base_conf.keys()[0])
                 base = base_table.get_exps(base_params)[0]
+
             # Write deadline misses / tardiness into result
             st.extract_sched_data(exp.data_files.st, result,
                                   base.data_files.st if base else None)
@@ -135,8 +137,7 @@ def main():
         sh.rmtree(opts.out_dir)
 
     # Remove un-plottable values
-    if opts.clean:
-        result_table.reduce()
+    result_table.reduce()
 
     result_table.write_result(opts.out_dir)