5 files changed, 102 insertions, 44 deletions

diff --git a/config/config.py b/config/config.py
index cbac6b2..1ac468b 100644
--- a/config/config.py
+++ b/config/config.py
@@ -39,13 +39,18 @@ DEFAULTS = {'params_file' : 'params.py',
             'sched_file'  : 'sched.py',
             'duration'    : 10,
             'prog'        : 'rtspin',
+            'out-gen'     : 'exps',
+            'out-run'     : 'run-data',
+            'out-parse'   : 'parse-data',
+            'out-plot'    : 'plot-data',
             'cycles'      : ft_freq() or 2000}
 
+
 '''Default sched_trace events (this is all of them).'''
 SCHED_EVENTS = range(501, 513)
 
 '''Overhead events.'''
-OVH_BASE_EVENTS  = ['SCHED', 'RELEASE', 'SCHED2', 'TICK', 'CXS']
+OVH_BASE_EVENTS  = ['SCHED', 'RELEASE', 'SCHED2', 'TICK', 'CXS', 'LOCK', 'UNLOCK']
 OVH_ALL_EVENTS   = ["%s_%s" % (e, t) for (e,t) in
                     itertools.product(OVH_BASE_EVENTS, ["START","END"])]
 OVH_ALL_EVENTS  += ['RELEASE_LATENCY']
diff --git a/gen_exps.py b/gen_exps.py
index 6488cdc..b847661 100755
--- a/gen_exps.py
+++ b/gen_exps.py
@@ -7,6 +7,7 @@ import re
 import shutil as sh
 import sys
 
+from config.config import DEFAULTS
 from optparse import OptionParser
 
 def parse_args():
@@ -15,7 +16,7 @@ def parse_args():
 
     parser.add_option('-o', '--out-dir', dest='out_dir',
                       help='directory for data output',
-                      default=("%s/exps"%os.getcwd()))
+                      default=("%s/%s"% (os.getcwd(), DEFAULTS['out-gen'])))
     parser.add_option('-f', '--force', action='store_true', default=False,
                       dest='force', help='overwrite existing data')
     parser.add_option('-n', '--num-trials', default=1, type='int', dest='trials',
@@ -51,9 +52,9 @@ def main():
     if opts.described != None:
         for generator in opts.described.split(','):
             if generator not in gen.get_generators():
-                print("No generator '%s'" % generator)
+                sys.stderr.write("No generator '%s'\n" % generator)
             else:
-                sys.stdout.write("Generator '%s', " % generator)
+                print("Generator '%s', " % generator)
                 gen.get_generators()[generator]().print_help()
     if opts.list_gens or opts.described:
         return 0
@@ -85,7 +86,7 @@ def main():
         if gen_name not in gen.get_generators():
             raise ValueError("Invalid generator '%s'" % gen_name)
 
-        print("Creating experiments using %s generator..." % gen_name)
+        sys.stderr.write("Creating experiments with %s generator...\n" % gen_name)
 
         params = dict(gen_params.items() + global_params.items())
         clazz  = gen.get_generators()[gen_name]
@@ -94,5 +95,7 @@ def main():
 
         generator.create_exps(opts.out_dir, opts.force, opts.trials)
 
+    sys.stderr.write("Experiments saved in %s.\n" % opts.out_dir)
+
 if __name__ == '__main__':
     main()
diff --git a/parse_exps.py b/parse_exps.py
index c254536..d07378c 100755
--- a/parse_exps.py
+++ b/parse_exps.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 from __future__ import print_function
 
-import config.config as conf
 import os
 import parse.ft as ft
 import parse.sched as st
@@ -12,6 +11,7 @@ import traceback
 
 from collections import namedtuple
 from common import load_params
+from config.config import DEFAULTS,PARAMS
 from optparse import OptionParser
 from parse.point import ExpPoint
 from parse.tuple_table import TupleTable
@@ -22,7 +22,8 @@ def parse_args():
     parser = OptionParser("usage: %prog [options] [data_dir]...")
 
     parser.add_option('-o', '--out', dest='out',
-                      help='file or directory for data output', default='parse-data')
+                      help='file or directory for data output',
+                      default=DEFAULTS['out-parse'])
     parser.add_option('-i', '--ignore', metavar='[PARAM...]', default="",
                       help='ignore changing parameter values')
     parser.add_option('-f', '--force', action='store_true', default=False,
@@ -41,7 +42,7 @@ def parse_args():
 ExpData = namedtuple('ExpData', ['path', 'params', 'work_dir'])
 
 def get_exp_params(data_dir, cm_builder):
-    param_file = "%s/%s" % (data_dir, conf.DEFAULTS['params_file'])
+    param_file = "%s/%s" % (data_dir, DEFAULTS['params_file'])
     if os.path.isfile(param_file):
         params = load_params(param_file)
 
@@ -53,8 +54,8 @@ def get_exp_params(data_dir, cm_builder):
          params = {}
 
     # Cycles must be present for feather-trace measurement parsing
-    if conf.PARAMS['cycles'] not in params:
-        params[conf.PARAMS['cycles']] = conf.DEFAULTS['cycles']
+    if PARAMS['cycles'] not in params:
+        params[PARAMS['cycles']] = DEFAULTS['cycles']
 
     return params
 
@@ -101,7 +102,7 @@ def parse_exp(exp_force):
     if not result:
         try:
             result = ExpPoint(exp.path)
-            cycles = exp.params[conf.PARAMS['cycles']]
+            cycles = exp.params[PARAMS['cycles']]
 
             # Write overheads into result
             ft.extract_ft_data(result, exp.path, exp.work_dir, cycles)
@@ -116,21 +117,31 @@ def parse_exp(exp_force):
 
     return (exp, result)
 
+def get_exps(args):
+    if args:
+        return args
+    elif os.path.exists(DEFAULTS['out-run']):
+        sys.stderr.write("Reading data from %s/*\n" % DEFAULTS['out-run'])
+        sched_dirs = os.listdir(DEFAULTS['out-run'])
+        return ['%s/%s' % (DEFAULTS['out-run'], d) for d in sched_dirs]
+    else:
+        sys.stderr.write("Reading data from current directory.\n")
+        return [os.getcwd()]
+
 def main():
     opts, args = parse_args()
-
-    args = args or [os.getcwd()]
+    exp_dirs  = get_exps(args)
 
     # Load exp parameters into a ColMap
     builder = ColMapBuilder()
-    exps = load_exps(args, builder, opts.force)
+    exps = load_exps(exp_dirs, builder, opts.force)
 
     # Don't track changes in ignored parameters
     if opts.ignore:
         for param in opts.ignore.split(","):
             builder.try_remove(param)
     # Always average multiple trials
-    builder.try_remove(conf.PARAMS['trial'])
+    builder.try_remove(PARAMS['trial'])
 
     col_map = builder.build()
     result_table = TupleTable(col_map)
@@ -175,7 +186,8 @@ def main():
         # No csvs to write, assume user meant to print out data
         if dir_map.is_empty():
             if not opts.verbose:
-                sys.stderr.write("Too little data to make csv files.\n")
+                sys.stderr.write("Too little data to make csv files, " +
+                                 "printing results.\n")
                 for key, exp in result_table:
                     for e in exp:
                         print(e)
diff --git a/plot_exps.py b/plot_exps.py
index 76e7396..15c54d0 100755
--- a/plot_exps.py
+++ b/plot_exps.py
@@ -6,7 +6,9 @@ import os
 import shutil as sh
 import sys
 import traceback
+
 from collections import namedtuple
+from config.config import DEFAULTS
 from multiprocessing import Pool, cpu_count
 from optparse import OptionParser
 from parse.col_map import ColMap,ColMapBuilder
@@ -17,7 +19,8 @@ def parse_args():
     parser = OptionParser("usage: %prog [options] [csv_dir]...")
 
     parser.add_option('-o', '--out-dir', dest='out_dir',
-                      help='directory for plot output', default='plot-data')
+                      help='directory for plot output',
+                      default=DEFAULTS['out-plot'])
     parser.add_option('-f', '--force', action='store_true', default=False,
                       dest='force', help='overwrite existing data')
     parser.add_option('-p', '--processors', default=max(cpu_count() - 1, 1),
@@ -139,21 +142,31 @@ def plot_dir(data_dir, out_dir, max_procs, force):
 
     sys.stderr.write('\n')
 
+def get_dirs(args):
+    if args:
+        return args
+    elif os.path.exists(DEFAULTS['out-parse']):
+        return [DEFAULTS['out-parse']]
+    else:
+        return os.getcwd()
+
 def main():
     opts, args = parse_args()
-    args = args or [os.getcwd()]
+    dirs = get_dirs(args)
 
     if opts.force and os.path.exists(opts.out_dir):
         sh.rmtree(opts.out_dir)
     if not os.path.exists(opts.out_dir):
         os.mkdir(opts.out_dir)
 
-    for dir in args:
-        if len(args) > 1:
+    for dir in dirs:
+        if len(dirs) > 1:
             out_dir = "%s/%s" % (opts.out_dir, os.path.split(dir)[1])
         else:
             out_dir = opts.out_dir
         plot_dir(dir, out_dir, opts.processors, opts.force)
 
+    sys.stderr.write("Plots saved in %s.\n" % opts.out_dir)
+
 if __name__ == '__main__':
     main()
diff --git a/run_exps.py b/run_exps.py
index 905d033..d7a06b5 100755
--- a/run_exps.py
+++ b/run_exps.py
@@ -2,13 +2,13 @@
 from __future__ import print_function
 
 import common as com
-import config.config as conf
 import os
 import re
 import shutil
 import sys
 import run.tracer as trace
 
+from config.config import PARAMS,DEFAULTS
 from collections import namedtuple
 from optparse import OptionParser
 from run.executable.executable import Executable
@@ -56,12 +56,13 @@ def parse_args():
     parser.add_option('-d', '--duration', dest='duration', type='int',
                       help='duration (seconds) of tasks')
     parser.add_option('-o', '--out-dir', dest='out_dir',
-                      help='directory for data output', default=("%s/run-data"%os.getcwd()))
+                      help='directory for data output',
+                      default=DEFAULTS['out-run'])
     parser.add_option('-p', '--params', dest='param_file',
                       help='file with experiment parameters')
     parser.add_option('-c', '--schedule-file', dest='sched_file',
                       help='name of schedule files within directories',
-                      default=conf.DEFAULTS['sched_file'])
+                      default=DEFAULTS['sched_file'])
     parser.add_option('-f', '--force', action='store_true', default=False,
                       dest='force', help='overwrite existing data')
     parser.add_option('-j', '--jabber', metavar='username@domain',
@@ -96,7 +97,7 @@ def convert_data(data):
             proc = (loc, match.group("CONTENT"))
             procs.append(proc)
         else:
-            prog = match.group("PROG") or conf.DEFAULTS['prog']
+            prog = match.group("PROG") or DEFAULTS['prog']
             spin = (prog, match.group("ARGS"))
             tasks.append(spin)
 
@@ -184,10 +185,10 @@ def verify_environment(exp_params):
 def run_parameter(exp_dir, out_dir, params, param_name):
     '''Run an executable (arguments optional) specified as a configurable
     @param_name in @params.'''
-    if conf.PARAMS[param_name] not in params:
+    if PARAMS[param_name] not in params:
         return
 
-    script_params = params[conf.PARAMS[param_name]]
+    script_params = params[PARAMS[param_name]]
 
     # Split into arguments and program name
     if type(script_params) != type([]):
@@ -210,22 +211,22 @@ def get_exp_params(cmd_scheduler, cmd_duration, file_params):
     '''Return ExpParam with configured values of all hardcoded params.'''
     kernel = copts = ""
 
-    scheduler = cmd_scheduler or file_params[conf.PARAMS['sched']]
-    duration  = cmd_duration  or file_params[conf.PARAMS['dur']] or\
-                conf.DEFAULTS['duration']
+    scheduler = cmd_scheduler or file_params[PARAMS['sched']]
+    duration  = cmd_duration  or file_params[PARAMS['dur']] or\
+                DEFAULTS['duration']
 
     # Experiments can specify required kernel name
-    if conf.PARAMS['kernel'] in file_params:
-        kernel = file_params[conf.PARAMS['kernel']]
+    if PARAMS['kernel'] in file_params:
+        kernel = file_params[PARAMS['kernel']]
 
     # Or required config options
-    if conf.PARAMS['copts'] in file_params:
-        copts = file_params[conf.PARAMS['copts']]
+    if PARAMS['copts'] in file_params:
+        copts = file_params[PARAMS['copts']]
 
     # Or required tracers
     requested = []
-    if conf.PARAMS['trace'] in file_params:
-        requested = file_params[conf.PARAMS['trace']]
+    if PARAMS['trace'] in file_params:
+        requested = file_params[PARAMS['trace']]
     tracers = trace.get_tracer_types(requested)
 
     # But only these two are mandatory
@@ -250,7 +251,7 @@ def load_experiment(sched_file, cmd_scheduler, cmd_duration,
 
     # Load parameter file
     param_file = param_file or \
-      "%s/%s" % (dir_name, conf.DEFAULTS['params_file'])
+      "%s/%s" % (dir_name, DEFAULTS['params_file'])
     if os.path.isfile(param_file):
         file_params = com.load_params(param_file)
     else:
@@ -277,19 +278,36 @@ def load_experiment(sched_file, cmd_scheduler, cmd_duration,
 
     # Save parameters used to run experiment in out_dir
     out_params = dict(file_params.items() +
-                      [(conf.PARAMS['sched'],  exp_params.scheduler),
-                       (conf.PARAMS['tasks'],  len(execs)),
-                       (conf.PARAMS['dur'],    exp_params.duration)])
+                      [(PARAMS['sched'],  exp_params.scheduler),
+                       (PARAMS['tasks'],  len(execs)),
+                       (PARAMS['dur'],    exp_params.duration)])
 
     # Feather-trace clock frequency saved for accurate overhead parsing
     ft_freq = com.ft_freq()
     if ft_freq:
-        out_params[conf.PARAMS['cycles']] = ft_freq
+        out_params[PARAMS['cycles']] = ft_freq
 
-    with open("%s/%s" % (out_dir, conf.DEFAULTS['params_file']), 'w') as f:
+    with open("%s/%s" % (out_dir, DEFAULTS['params_file']), 'w') as f:
         f.write(str(out_params))
 
 
+def get_exps(opts, args):
+    if args:
+        return args
+
+    # Default to sched_file > generated dirs
+    if os.path.exists(opts.sched_file):
+        sys.stderr.write("Reading schedule from %s.\n" % opts.sched_file)
+        return [opts.sched_file]
+    elif os.path.exists(DEFAULTS['out-gen']):
+        sys.stderr.write("Reading schedules from %s/*.\n" % DEFAULTS['out-gen'])
+        sched_dirs = os.listdir(DEFAULTS['out-gen'])
+        return ['%s/%s' % (DEFAULTS['out-gen'], d) for d in sched_dirs]
+    else:
+        sys.stderr.write("Run with -h to view options.\n");
+        sys.exit(1)
+
+
 def setup_jabber(target):
     try:
         from run.jabber import Jabber
@@ -300,6 +318,7 @@ def setup_jabber(target):
                          "Disabling instant messages.\n")
         return None
 
+
 def setup_email(target):
     try:
         from run.emailer import Emailer
@@ -313,6 +332,7 @@ def setup_email(target):
     sys.stderr.write(message + " Disabling email message.\n")
     return None
 
+
 def main():
     opts, args = parse_args()
 
@@ -321,7 +341,7 @@ def main():
     param_file = opts.param_file
     out_base   = os.path.abspath(opts.out_dir)
 
-    args = args or [opts.sched_file]
+    exps = get_exps(opts, args)
 
     created = False
     if not os.path.exists(out_base):
@@ -335,9 +355,9 @@ def main():
     invalid = 0
 
     jabber = setup_jabber(opts.jabber) if opts.jabber else None
-    email  = setup_email(opts.email) if opts.email else None
+    email  = setup_email(opts.email)   if opts.email  else None
 
-    for exp in args:
+    for exp in exps:
         path = "%s/%s" % (os.getcwd(), exp)
         out_dir = "%s/%s" % (out_base, os.path.split(exp.strip('/'))[1])
 
@@ -374,6 +394,7 @@ def main():
 
         ran += 1
 
+    # Clean out directory if it failed immediately
     if not os.listdir(out_base) and created and not succ:
         os.rmdir(out_base)
 
@@ -385,6 +406,10 @@ def main():
 
     print(message)
 
+    if succ:
+        sys.stderr.write("Successful experiment data saved in %s.\n" %
+                         opts.out_dir)
+
     if email:
         email.send(message)
         email.close()