Removed 2-step parse for overheads.

4 files changed, 56 insertions, 104 deletions

diff --git a/parse/ft.py b/parse/ft.py
index cbf75f2..c5f1522 100644
--- a/parse/ft.py
+++ b/parse/ft.py
@@ -7,94 +7,68 @@ import subprocess
 
 from point import Measurement,Type
 
-SPLIT_DATA_NAME = "overhead={}.bin"
-FT_DATA_NAME    = "sorted-ft.bin"
-FIELDS = ["Overhead", "samples", "max", "avg", "min", "med", "std", "var"]
+FT_SPLIT_NAME  = "overhead={}.bin"
+FT_SORTED_NAME = "sorted-ft.bin"
+FT_ERR_NAME    = "err-ft"
 
-def get_ft_output(data_dir, cycles, out_dir, force=False):
-    """
-    Create and return file containing analyzed overhead data
-    """
+def extract_ft_data(result, data_dir, cycles, tmp_dir):
     freg = conf.FILES['ft_data'] + "$"
     bins = [f for f in os.listdir(data_dir) if re.match(freg, f)]
 
-    output_file  = "{}/out-ft".format(out_dir)
+    if not len(bins):
+        return False
 
-    if os.path.isfile(output_file):
-        if force:
-            os.remove(output_file)
-        else:
-            return output_file
+    bin_file = "{}/{}".format(data_dir, bins[0])
 
-    if len(bins) != 0:
-        bin_file = "{}/{}".format(data_dir, bins[0])
-        err_file = open("%s/err-ft" % out_dir, 'w')
+    with open("%s/%s" % (tmp_dir, FT_ERR_NAME), 'w') as err_file:
+        sorted_bin = sort_ft(bin_file, err_file, tmp_dir)
 
-        sorted_bin = sort_ft(bin_file, err_file, out_dir)
-        make_data_file(sorted_bin, cycles, output_file, err_file, out_dir)
+        for event in conf.BASE_EVENTS:
+            parse_overhead(result, sorted_bin, event, cycles,
+                           tmp_dir, err_file)
 
         os.remove(sorted_bin)
 
-        return output_file
-    else:
-        return None
-    return output_file
+    return True
 
-def fmt_cell(x):
-    if type(x) == str:
-        return "%15s" % x
-    if type(x) == int:
-        return "%15d" % x
-    else:
-        return "%15.3f" % x
+def parse_overhead(result, overhead_bin, overhead, cycles, out_dir, err_file):
+    ovh_fname = "{}/{}".format(out_dir, FT_SPLIT_NAME).format(overhead)
 
-def make_data_file(sorted_bin, cycles, out_fname, err_file, out_dir):
-    """
-    Create file containing all overhead information.
-    """
-    base_name = "{}/{}".format(out_dir, SPLIT_DATA_NAME)
+    if os.path.exists(ovh_fname):
+        os.remove(ovh_fname)
+    ovh_file = open(ovh_fname, 'w')
 
-    with open(out_fname, "w") as f:
-        f.write("#%s" % ", ".join(fmt_cell(x) for x in FIELDS))
-        f.write("\n")
+    # Extract matching overhead events into a seperate file
+    cmd  = [conf.BINS["split"], "-r", "-b", overhead, overhead_bin]
+    ret  = subprocess.call(cmd, cwd=out_dir, stderr=err_file, stdout=ovh_file)
+    size = os.stat(ovh_fname).st_size
 
-        for event in conf.BASE_EVENTS:
-                ovh_fname = base_name.format(event.replace("_", "-"))
-
-                if os.path.exists(ovh_fname):
-                    os.remove(ovh_fname)
-                ovh_file = open(ovh_fname, 'w')
-
-                # Extract matching overhead events into a seperate file
-                cmd = [conf.BINS["split"], "-r", "-b", event, sorted_bin]
-                ret = subprocess.call(cmd, cwd=out_dir,
-                                      stderr=err_file, stdout=ovh_file)
-                size = os.stat(ovh_fname).st_size
+    if ret:
+        raise Exception("Failed (%d) with command: %s" % (ret, " ".join(cmd)))
+    if not size:
+        os.remove(ovh_fname)
 
-                if ret:
-                    err_file.write("Failed with command: %s" % " ".join(cmd))
-                if not size:
-                    os.remove(ovh_fname)
-                if not size or ret:
-                    continue
+    if size and not ret:
+        # Map and sort file for stats
+        data = np.memmap(ovh_fname, dtype="float32", mode='c')
+        data /= float(cycles) # Scale for processor speed
+        data.sort()
 
-                # Map and sort file for stats
-                data = np.memmap(ovh_fname, dtype="float32", mode='c')
-                data /= float(cycles) # Scale for processor speed
-                data.sort()
+        m = Measurement("%s-%s" % (overhead_bin, overhead))
+        m[Type.Max] = data[-1]
+        m[Type.Avg] = np.mean(data)
+        m[Type.Min] = data[0]
+        m[Type.Var] = np.var(data)
 
-                stats = [event, len(data), data[-1], np.mean(data), data[0],
-                         np.median(data), np.std(data, ddof=1), np.var(data)]
-                f.write(", ".join([fmt_cell(x) for x in stats]))
-                f.write("\n")
+        result[overhead] = m
 
-                os.remove(ovh_fname)
+        os.remove(ovh_fname)
 
 def sort_ft(ft_file, err_file, out_dir):
     """
     Create and return file with sorted overheads from @ft_file.
     """
-    out_fname = "{}/{}".format(out_dir, FT_DATA_NAME)
+    out_fname = "{}/{}".format(out_dir, FT_SORTED_NAME)
 
     # Sort happens in-place
     sh.copyfile(ft_file, out_fname)
@@ -105,28 +79,3 @@ def sort_ft(ft_file, err_file, out_dir):
         raise Exception("Sort failed with command: %s" % " ".join(cmd))
 
     return out_fname
-
-def extract_ft_data(data_file, result, overheads):
-    """
-    Return exp point with overhead measurements from data_file
-    """
-    with open(data_file) as f:
-        data = f.read()
-
-    for ovh in overheads:
-        regex = r"({}[^\n]*)".format(ovh)
-        line = re.search(regex, data)
-
-        if not line:
-            continue
-
-        vals = re.split(r"[,\s]+", line.groups(1)[0])
-
-        measure = Measurement("%s-%s" % (data_file, ovh))
-        measure[Type.Max] = float(vals[FIELDS.index("max")])
-        measure[Type.Avg] = float(vals[FIELDS.index("avg")])
-        measure[Type.Var] = float(vals[FIELDS.index("var")])
-
-        result[ovh] = measure
-
-    return result
diff --git a/parse/sched.py b/parse/sched.py
index bbf6e10..3e30880 100644
--- a/parse/sched.py
+++ b/parse/sched.py
@@ -11,7 +11,6 @@ import os
 import re
 import numpy as np
 import subprocess
-import pprint
 
 from collections import namedtuple,defaultdict
 from operator import methodcaller
diff --git a/parse_exps.py b/parse_exps.py
index 24bdb85..d932b0d 100755
--- a/parse_exps.py
+++ b/parse_exps.py
@@ -42,7 +42,7 @@ def parse_args():
     return parser.parse_args()
 
 ExpData   = namedtuple('ExpData', ['name', 'params', 'data_files', 'is_base'])
-DataFiles = namedtuple('DataFiles', ['ft','st'])
+DataFiles = namedtuple('DataFiles', ['st'])
 
 def get_exp_params(data_dir, col_map):
     param_file = "%s/%s" % (data_dir, conf.DEFAULTS['params_file'])
@@ -82,10 +82,7 @@ def gen_exp_data(exp_dirs, base_conf, col_map, force):
 
         # Read and translate exp output files
         params = get_exp_params(data_dir, col_map)
-        cycles = int(params[conf.PARAMS['cycles']])
         st_output = st.get_st_output(data_dir, tmp_dir, force)
-        ft_output = ft.get_ft_output(data_dir, cycles, tmp_dir, force)
-
 
         if base_conf and base_conf.viewitems() & params.viewitems():
             if not st_output:
@@ -97,14 +94,14 @@ def gen_exp_data(exp_dirs, base_conf, col_map, force):
             base_params.pop(base_conf.keys()[0])
 
             base_exp = ExpData(data_dir, base_params,
-                               DataFiles(ft_output, st_output), True)
+                               DataFiles(st_output), True)
             scaling_bases += [base_exp]
         else:
             is_base = False
 
         # Create experiment named after the data dir
         exp_data = ExpData(data_dir, params,
-                           DataFiles(ft_output, st_output), is_base)
+                           DataFiles(st_output), is_base)
 
         plain_exps += [exp_data]
 
@@ -142,13 +139,20 @@ def main():
         base_table.add_exp(base.params, base)
 
     sys.stderr.write("Parsing data...\n")
+    for exp in args:
+        result = ExpPoint(exp)
+        params = get_exp_params(exp, col_map)
+        # Write overheads into result
+        ft.extract_ft_data(result, exp,
+                           params[conf.PARAMS['cycles']],
+                           exp + "/tmp")
+        
+        if opts.verbose:
+            print(result)
+
     for i,exp in enumerate(plain_exps):
         result = ExpPoint(exp.name)
 
-        if exp.data_files.ft:
-            # Write overheads into result
-            ft.extract_ft_data(exp.data_files.ft, result, conf.BASE_EVENTS)
-
         if exp.data_files.st:
             base = None
             if base_conf and not exp.is_base:
diff --git a/run_exps.py b/run_exps.py
index 3efb09d..1d2cc2e 100755
--- a/run_exps.py
+++ b/run_exps.py
@@ -67,8 +67,8 @@ def convert_data(data):
     return {'proc' : procs, 'spin' : spins}
 
 def fix_paths(schedule, exp_dir, sched_file):
+    """Replace relative paths of command line arguments with absolute ones."""
     for (idx, (spin, args)) in enumerate(schedule['spin']):
-        # Replace relative paths (if present) with absolute ones
         for arg in re.split(" +", args):
             abspath = "%s/%s" % (exp_dir, arg)
             if os.path.exists(abspath):
@@ -136,8 +136,8 @@ def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir):
     proc_entries = []
     executables  = []
 
-    # if kernel and not lu.uname_matches(kernel):
-    #     raise InvalidKernel(kernel)
+    if kernel and not lu.uname_matches(kernel):
+        raise InvalidKernel(kernel)
 
     # Parse values for proc entries
     for entry_conf in schedule['proc']: