7 files changed, 79 insertions, 44 deletions

diff --git a/experiment/experiment.py b/experiment/experiment.py
index e6dc92d..f0c201d 100644
--- a/experiment/experiment.py
+++ b/experiment/experiment.py
@@ -197,7 +197,7 @@ class Experiment(object):
         self.exec_out and self.exec_out.close()
         self.exec_err and self.exec_err.close()
 
-        sleep_time = 5
+        sleep_time = 10
         self.log("Sleeping %d seconds to allow buffer flushing" % sleep_time)
         time.sleep(sleep_time)
 
diff --git a/parse/ft.py b/parse/ft.py
index 127e49f..c915978 100644
--- a/parse/ft.py
+++ b/parse/ft.py
@@ -6,7 +6,10 @@ import subprocess
 
 from point import Measurement,Type
 
-def get_ft_output(data_dir, out_dir):
+def get_ft_output(data_dir, out_dir, force=False):
+    """
+    Create and return files containing sorted and analyzed overhead data
+    """
     bin_file = conf.FILES['ft_data'] + "$"
     bins = [f for f in os.listdir(data_dir) if re.match(bin_file, f)]
 
@@ -14,8 +17,11 @@ def get_ft_output(data_dir, out_dir):
     output_file  = "{}/out-ft".format(out_dir)
 
     if os.path.isfile(output_file):
-        print("ft-output already exists for %s" % data_dir)
-        return output_file
+        if force:
+            os.remove(output_file)
+        else:
+            print("ft-output already exists for %s" % data_dir)
+            return output_file
 
     if len(bins) != 0:
         err_file = open("%s/err-ft" % out_dir, 'w')
diff --git a/parse/sched.py b/parse/sched.py
index 300c569..a65f001 100644
--- a/parse/sched.py
+++ b/parse/sched.py
@@ -50,7 +50,7 @@ class LeveledArray(object):
             name = "%s%s" % ("%s-" % level if level else "", self.name)
             result[name] = Measurement(name).from_array(arr)
 
-def get_st_output(data_dir, out_dir):
+def get_st_output(data_dir, out_dir, force=False):
     """
     Create and return files containing unpacked sched data
     """
@@ -60,7 +60,11 @@ def get_st_output(data_dir, out_dir):
     output_file = "%s/out-st" % out_dir
 
     if os.path.isfile(output_file):
-        return output_file
+        if force:
+            os.remove(output_file)
+        else:
+            print("st-output already exists for %s" % data_dir)
+            return output_file
 
     if len(bins) != 0:
         cmd_arr = [conf.BINS['st_show']]
diff --git a/parse/tuple_table.py b/parse/tuple_table.py
index e6f0cc5..5e98d87 100644
--- a/parse/tuple_table.py
+++ b/parse/tuple_table.py
@@ -24,7 +24,7 @@ class ColMap(object):
         if added < len(kv):
             raise Exception("column map '%s' missed field in map '%s'" %
                             (self.col_list, kv))
-        
+
         return key
 
     def __contains__(self, col):
@@ -58,7 +58,7 @@ class TupleTable(object):
         key = self.col_map.get_key(kv)
         return self.table[key]
 
-    def __reduce(self):
+    def reduce(self):
         if self.reduced:
             raise Exception("cannot reduce twice!")
         self.reduced = True
@@ -67,7 +67,6 @@ class TupleTable(object):
 
     def write_result(self, out_dir):
         dir_map = DirMap(out_dir)
-        self.__reduce()
         for key, point in self.table.iteritems():
             kv = self.col_map.get_map(key)
 
diff --git a/parse_exps.py b/parse_exps.py
index 8f98309..1f36bab 100755
--- a/parse_exps.py
+++ b/parse_exps.py
@@ -7,6 +7,7 @@ import os
 import parse.ft as ft
 import parse.sched as st
 import re
+import shutil as sh
 
 from collections import namedtuple
 from common import load_params
@@ -18,10 +19,16 @@ def parse_args():
     parser = OptionParser("usage: %prog [options] [data_dir]...")
 
     parser.add_option('-o', '--out-dir', dest='out_dir',
-                      help='directory for data output', default=os.getcwd())
+                      help='directory for data output', default='parse-data')
+    parser.add_option('-c', '--clean', action='store_true', default=False,
+                      dest='clean', help='do not output single-point csvs')
     parser.add_option('-s', '--scale-against', dest='scale_against',
                       metavar='PARAM=VALUE', default="",
                       help='calculate task scaling factors against these configs')
+    parser.add_option('-f', '--force', action='store_true', default=False,
+                      dest='force', help='overwrite existing data')
+    parser.add_option('-v', '--verbose', action='store_true', default=False,
+                      dest='verbose', help='print out data points')
 
     return parser.parse_args()
 
@@ -46,7 +53,7 @@ def get_exp_params(data_dir, col_map):
     return params
 
 
-def gen_exp_data(exp_dirs, base_conf, col_map):
+def gen_exp_data(exp_dirs, base_conf, col_map, force):
     plain_exps = []
     scaling_bases  = []
 
@@ -60,8 +67,8 @@ def gen_exp_data(exp_dirs, base_conf, col_map):
 
         # Read and translate exp output files
         params = get_exp_params(data_dir, col_map)
-        st_output = st.get_st_output(data_dir, tmp_dir)
-        ft_output = ft.get_ft_output(data_dir, tmp_dir)
+        st_output = st.get_st_output(data_dir, tmp_dir, force)
+        ft_output = ft.get_ft_output(data_dir, tmp_dir, force)
 
         # Create experiment named after the data dir
         exp_data = ExpData(data_dir, params, DataFiles(ft_output, st_output))
@@ -88,7 +95,7 @@ def main():
 
     col_map = ColMap()
 
-    (plain_exps, scaling_bases) = gen_exp_data(args, base_conf, col_map)
+    (plain_exps, scaling_bases) = gen_exp_data(args, base_conf, col_map, opts.force)
 
     if base_conf and base_conf.keys()[0] not in col_map:
         raise IOError("Base column '%s' not present in any parameters!" %
@@ -121,8 +128,15 @@ def main():
 
         result_table.add_exp(exp.params, result)
 
-        print(result)
+        if opts.verbose:
+            print(result)
 
+    if opts.force and os.path.exists(opts.out_dir):
+        sh.rmtree(opts.out_dir)
+
+    # Remove un-plottable values
+    if opts.clean:
+        result_table.reduce()
 
     result_table.write_result(opts.out_dir)
 
diff --git a/plot_exps.py b/plot_exps.py
index e69de29..06f43b0 100644
--- a/plot_exps.py
+++ b/plot_exps.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+from optparse import OptionParser
+
+def parse_args():
+    parser = OptionParser("usage: %prog [options] [csv_dir]...")
+
+    parser.add_option('-o', '--out-dir', dest='out_dir',
+                      help='directory for plot output', default='plot-data')
+    parser.add_option('-f', '--force', action='store_true', default=False,
+                      dest='force', help='overwrite existing data')
+
+    return parser.parse_args()
+
+def main():
+    opts, args = parse_args()
+    args = args or [os.getcwd()]
+
+if __name__ == '__main__':
+    main()
diff --git a/run_exps.py b/run_exps.py
index 8f72adb..19dbad1 100755
--- a/run_exps.py
+++ b/run_exps.py
@@ -1,14 +1,11 @@
 #!/usr/bin/env python
 from __future__ import print_function
-"""
-TODO: no -f flag, instead allow individual schedules to be passed in.
-      -f flag now forced, which removes old data directories
-"""
 
 import config.config as conf
 import experiment.litmus_util as lu
 import os
 import re
+import shutil
 import traceback
 
 from common import load_params
@@ -22,19 +19,21 @@ def InvalidKernel(Exception):
         self.kernel = kernel
 
 def parse_args():
-    parser = OptionParser("usage: %prog [options] [sched_file]... [exp_dir]...]")
+    parser = OptionParser("usage: %prog [options] [sched_file]... [exp_dir]...")
 
     parser.add_option('-s', '--scheduler', dest='scheduler',
                       help='scheduler for all experiments')
     parser.add_option('-d', '--duration', dest='duration', type='int',
                       help='duration (seconds) of tasks')
     parser.add_option('-o', '--out-dir', dest='out_dir',
-                      help='directory for data output', default=os.getcwd())
+                      help='directory for data output', default="run-data")
     parser.add_option('-p', '--params', dest='param_file',
                       help='file with experiment parameters')
-    parser.add_option('-f', '--schedule-file', dest='sched_file',
-                      help='name of schedule files',
+    parser.add_option('-c', '--schedule-file', dest='sched_file',
+                      help='name of schedule files within directories',
                       default=conf.DEFAULTS['sched_file'])
+    parser.add_option('-f', '--force', action='store_true', default=False,
+                      dest='force', help='overwrite existing data')
 
     return parser.parse_args()
 
@@ -67,33 +66,21 @@ def convert_data(data):
 
     return {'proc' : procs, 'spin' : spins}
 
-def fix_paths(schedule, exp_dir):
+def fix_paths(schedule, exp_dir, sched_file):
     for (idx, (spin, args)) in enumerate(schedule['spin']):
         # Replace relative paths (if present) with absolute ones
-        for arg in args.split(" "):
+        for arg in re.split(" +", args):
             abspath = "%s/%s" % (exp_dir, arg)
             if os.path.exists(abspath):
                 args = args.replace(arg, abspath)
                 break
+            elif re.match(r'.*\w+\.\w+', arg):
+                print("WARNING: non-existent file '%s' may be referenced:\n\t%s"
+                      % (arg, sched_file))
 
         schedule['spin'][idx] = (spin, args)
 
-def get_dirs(sched_file, out_base_dir):
-    sched_leaf_dir  = re.findall(r".*/([\w_-]+)/.*?$", sched_file)[0]
-    sched_full_dir = os.path.split(sched_file)[0]
-
-    work_dir = "%s/tmp" % sched_full_dir
-
-    if sched_full_dir == out_base_dir:
-        out_dir = "%s/data" % sched_full_dir
-    else:
-        # Put it under the base output dir with the same directory name
-        out_dir  = "%s/%s" % (out_base_dir, sched_leaf_dir)
-
-    return (work_dir, out_dir)
-
-
-def load_experiment(sched_file, scheduler, duration, param_file, out_base):
+def load_experiment(sched_file, scheduler, duration, param_file, out_dir):
     if not os.path.isfile(sched_file):
         raise IOError("Cannot find schedule file: %s" % sched_file)
 
@@ -121,8 +108,8 @@ def load_experiment(sched_file, scheduler, duration, param_file, out_base):
 
     # Parse schedule file's intentions
     schedule = load_schedule(sched_file)
-    (work_dir, out_dir) = get_dirs(sched_file, out_base)
-    fix_paths(schedule, os.path.split(sched_file)[0])
+    work_dir = "%s/tmp" % dirname
+    fix_paths(schedule, os.path.split(sched_file)[0], sched_file)
 
     run_exp(sched_file, schedule, scheduler, kernel, duration, work_dir, out_dir)
 
@@ -209,15 +196,19 @@ def main():
 
     for exp in args:
         path = "%s/%s" % (os.getcwd(), exp)
+        out_dir = "%s/%s" % (out_base, exp)
 
         if not os.path.exists(path):
             raise IOError("Invalid experiment: %s" % path)
 
+        if opts.force and os.path.exists(out_dir):
+            shutil.rmtree(out_dir)
+
         if os.path.isdir(exp):
             path = "%s/%s" % (path, opts.sched_file)
 
         try:
-            load_experiment(path, scheduler, duration, param_file, out_base)
+            load_experiment(path, scheduler, duration, param_file, out_dir)
             succ += 1
         except ExperimentDone:
             done += 1