Removed 2-step parse for overheads.

2 files changed, 39 insertions, 91 deletions

diff --git a/parse/ft.py b/parse/ft.py
index cbf75f2..c5f1522 100644
--- a/parse/ft.py
+++ b/parse/ft.py
@@ -7,94 +7,68 @@ import subprocess
 
 from point import Measurement,Type
 
-SPLIT_DATA_NAME = "overhead={}.bin"
-FT_DATA_NAME    = "sorted-ft.bin"
-FIELDS = ["Overhead", "samples", "max", "avg", "min", "med", "std", "var"]
+FT_SPLIT_NAME  = "overhead={}.bin"
+FT_SORTED_NAME = "sorted-ft.bin"
+FT_ERR_NAME    = "err-ft"
 
-def get_ft_output(data_dir, cycles, out_dir, force=False):
-    """
-    Create and return file containing analyzed overhead data
-    """
+def extract_ft_data(result, data_dir, cycles, tmp_dir):
     freg = conf.FILES['ft_data'] + "$"
     bins = [f for f in os.listdir(data_dir) if re.match(freg, f)]
 
-    output_file  = "{}/out-ft".format(out_dir)
+    if not len(bins):
+        return False
 
-    if os.path.isfile(output_file):
-        if force:
-            os.remove(output_file)
-        else:
-            return output_file
+    bin_file = "{}/{}".format(data_dir, bins[0])
 
-    if len(bins) != 0:
-        bin_file = "{}/{}".format(data_dir, bins[0])
-        err_file = open("%s/err-ft" % out_dir, 'w')
+    with open("%s/%s" % (tmp_dir, FT_ERR_NAME), 'w') as err_file:
+        sorted_bin = sort_ft(bin_file, err_file, tmp_dir)
 
-        sorted_bin = sort_ft(bin_file, err_file, out_dir)
-        make_data_file(sorted_bin, cycles, output_file, err_file, out_dir)
+        for event in conf.BASE_EVENTS:
+            parse_overhead(result, sorted_bin, event, cycles,
+                           tmp_dir, err_file)
 
         os.remove(sorted_bin)
 
-        return output_file
-    else:
-        return None
-    return output_file
+    return True
 
-def fmt_cell(x):
-    if type(x) == str:
-        return "%15s" % x
-    if type(x) == int:
-        return "%15d" % x
-    else:
-        return "%15.3f" % x
+def parse_overhead(result, overhead_bin, overhead, cycles, out_dir, err_file):
+    ovh_fname = "{}/{}".format(out_dir, FT_SPLIT_NAME).format(overhead)
 
-def make_data_file(sorted_bin, cycles, out_fname, err_file, out_dir):
-    """
-    Create file containing all overhead information.
-    """
-    base_name = "{}/{}".format(out_dir, SPLIT_DATA_NAME)
+    if os.path.exists(ovh_fname):
+        os.remove(ovh_fname)
+    ovh_file = open(ovh_fname, 'w')
 
-    with open(out_fname, "w") as f:
-        f.write("#%s" % ", ".join(fmt_cell(x) for x in FIELDS))
-        f.write("\n")
+    # Extract matching overhead events into a seperate file
+    cmd  = [conf.BINS["split"], "-r", "-b", overhead, overhead_bin]
+    ret  = subprocess.call(cmd, cwd=out_dir, stderr=err_file, stdout=ovh_file)
+    size = os.stat(ovh_fname).st_size
 
-        for event in conf.BASE_EVENTS:
-                ovh_fname = base_name.format(event.replace("_", "-"))
-
-                if os.path.exists(ovh_fname):
-                    os.remove(ovh_fname)
-                ovh_file = open(ovh_fname, 'w')
-
-                # Extract matching overhead events into a seperate file
-                cmd = [conf.BINS["split"], "-r", "-b", event, sorted_bin]
-                ret = subprocess.call(cmd, cwd=out_dir,
-                                      stderr=err_file, stdout=ovh_file)
-                size = os.stat(ovh_fname).st_size
+    if ret:
+        raise Exception("Failed (%d) with command: %s" % (ret, " ".join(cmd)))
+    if not size:
+        os.remove(ovh_fname)
 
-                if ret:
-                    err_file.write("Failed with command: %s" % " ".join(cmd))
-                if not size:
-                    os.remove(ovh_fname)
-                if not size or ret:
-                    continue
+    if size and not ret:
+        # Map and sort file for stats
+        data = np.memmap(ovh_fname, dtype="float32", mode='c')
+        data /= float(cycles) # Scale for processor speed
+        data.sort()
 
-                # Map and sort file for stats
-                data = np.memmap(ovh_fname, dtype="float32", mode='c')
-                data /= float(cycles) # Scale for processor speed
-                data.sort()
+        m = Measurement("%s-%s" % (overhead_bin, overhead))
+        m[Type.Max] = data[-1]
+        m[Type.Avg] = np.mean(data)
+        m[Type.Min] = data[0]
+        m[Type.Var] = np.var(data)
 
-                stats = [event, len(data), data[-1], np.mean(data), data[0],
-                         np.median(data), np.std(data, ddof=1), np.var(data)]
-                f.write(", ".join([fmt_cell(x) for x in stats]))
-                f.write("\n")
+        result[overhead] = m
 
-                os.remove(ovh_fname)
+        os.remove(ovh_fname)
 
 def sort_ft(ft_file, err_file, out_dir):
     """
     Create and return file with sorted overheads from @ft_file.
     """
-    out_fname = "{}/{}".format(out_dir, FT_DATA_NAME)
+    out_fname = "{}/{}".format(out_dir, FT_SORTED_NAME)
 
     # Sort happens in-place
     sh.copyfile(ft_file, out_fname)
@@ -105,28 +79,3 @@ def sort_ft(ft_file, err_file, out_dir):
         raise Exception("Sort failed with command: %s" % " ".join(cmd))
 
     return out_fname
-
-def extract_ft_data(data_file, result, overheads):
-    """
-    Return exp point with overhead measurements from data_file
-    """
-    with open(data_file) as f:
-        data = f.read()
-
-    for ovh in overheads:
-        regex = r"({}[^\n]*)".format(ovh)
-        line = re.search(regex, data)
-
-        if not line:
-            continue
-
-        vals = re.split(r"[,\s]+", line.groups(1)[0])
-
-        measure = Measurement("%s-%s" % (data_file, ovh))
-        measure[Type.Max] = float(vals[FIELDS.index("max")])
-        measure[Type.Avg] = float(vals[FIELDS.index("avg")])
-        measure[Type.Var] = float(vals[FIELDS.index("var")])
-
-        result[ovh] = measure
-
-    return result
diff --git a/parse/sched.py b/parse/sched.py
index bbf6e10..3e30880 100644
--- a/parse/sched.py
+++ b/parse/sched.py
@@ -11,7 +11,6 @@ import os
 import re
 import numpy as np
 import subprocess
-import pprint
 
 from collections import namedtuple,defaultdict
 from operator import methodcaller