Scripts read directories created by other scripts if no arguments.

With no arguments, all scripts first try to load the current directory. If the
current directory has no data, the scripts search for the output of the previous
scripts in the toolchain, e.g. parse_exps.py loads run-data/*, created by
run_exps.py.

This commit also switched messages to stderr where they belong, and adds in
missing lock and unlock overheads.

5 files changed, 102 insertions, 44 deletions

diff --git a/config/config.py b/config/config.py
index cbac6b2..1ac468b 100644
--- a/config/config.py
+++ b/config/config.py
@@ -39,13 +39,18 @@ DEFAULTS = {'params_file' : 'params.py',
             'sched_file'  : 'sched.py',
             'duration'    : 10,
             'prog'        : 'rtspin',
+            'out-gen'     : 'exps',
+            'out-run'     : 'run-data',
+            'out-parse'   : 'parse-data',
+            'out-plot'    : 'plot-data',
             'cycles'      : ft_freq() or 2000}
 
+
 '''Default sched_trace events (this is all of them).'''
 SCHED_EVENTS = range(501, 513)
 
 '''Overhead events.'''
-OVH_BASE_EVENTS  = ['SCHED', 'RELEASE', 'SCHED2', 'TICK', 'CXS']
+OVH_BASE_EVENTS  = ['SCHED', 'RELEASE', 'SCHED2', 'TICK', 'CXS', 'LOCK', 'UNLOCK']
 OVH_ALL_EVENTS   = ["%s_%s" % (e, t) for (e,t) in
                     itertools.product(OVH_BASE_EVENTS, ["START","END"])]
 OVH_ALL_EVENTS  += ['RELEASE_LATENCY']
diff --git a/gen_exps.py b/gen_exps.py
index 6488cdc..b847661 100755
--- a/gen_exps.py
+++ b/gen_exps.py
@@ -7,6 +7,7 @@ import re
 import shutil as sh
 import sys
 
+from config.config import DEFAULTS
 from optparse import OptionParser
 
 def parse_args():
@@ -15,7 +16,7 @@ def parse_args():
 
     parser.add_option('-o', '--out-dir', dest='out_dir',
                       help='directory for data output',
-                      default=("%s/exps"%os.getcwd()))
+                      default=("%s/%s"% (os.getcwd(), DEFAULTS['out-gen'])))
     parser.add_option('-f', '--force', action='store_true', default=False,
                       dest='force', help='overwrite existing data')
     parser.add_option('-n', '--num-trials', default=1, type='int', dest='trials',
@@ -51,9 +52,9 @@ def main():
     if opts.described != None:
         for generator in opts.described.split(','):
             if generator not in gen.get_generators():
-                print("No generator '%s'" % generator)
+                sys.stderr.write("No generator '%s'\n" % generator)
             else:
-                sys.stdout.write("Generator '%s', " % generator)
+                print("Generator '%s', " % generator)
                 gen.get_generators()[generator]().print_help()
     if opts.list_gens or opts.described:
         return 0
@@ -85,7 +86,7 @@ def main():
         if gen_name not in gen.get_generators():
             raise ValueError("Invalid generator '%s'" % gen_name)
 
-        print("Creating experiments using %s generator..." % gen_name)
+        sys.stderr.write("Creating experiments with %s generator...\n" % gen_name)
 
         params = dict(gen_params.items() + global_params.items())
         clazz  = gen.get_generators()[gen_name]
@@ -94,5 +95,7 @@ def main():
 
         generator.create_exps(opts.out_dir, opts.force, opts.trials)
 
+    sys.stderr.write("Experiments saved in %s.\n" % opts.out_dir)
+
 if __name__ == '__main__':
     main()
diff --git a/parse_exps.py b/parse_exps.py
index c254536..d07378c 100755
--- a/parse_exps.py
+++ b/parse_exps.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 from __future__ import print_function
 
-import config.config as conf
 import os
 import parse.ft as ft
 import parse.sched as st
@@ -12,6 +11,7 @@ import traceback
 
 from collections import namedtuple
 from common import load_params
+from config.config import DEFAULTS,PARAMS
 from optparse import OptionParser
 from parse.point import ExpPoint
 from parse.tuple_table import TupleTable
@@ -22,7 +22,8 @@ def parse_args():
     parser = OptionParser("usage: %prog [options] [data_dir]...")
 
     parser.add_option('-o', '--out', dest='out',
-                      help='file or directory for data output', default='parse-data')
+                      help='file or directory for data output',
+                      default=DEFAULTS['out-parse'])
     parser.add_option('-i', '--ignore', metavar='[PARAM...]', default="",
                       help='ignore changing parameter values')
     parser.add_option('-f', '--force', action='store_true', default=False,
@@ -41,7 +42,7 @@ def parse_args():
 ExpData = namedtuple('ExpData', ['path', 'params', 'work_dir'])
 
 def get_exp_params(data_dir, cm_builder):
-    param_file = "%s/%s" % (data_dir, conf.DEFAULTS['params_file'])
+    param_file = "%s/%s" % (data_dir, DEFAULTS['params_file'])
     if os.path.isfile(param_file):
         params = load_params(param_file)
 
@@ -53,8 +54,8 @@ def get_exp_params(data_dir, cm_builder):
          params = {}
 
     # Cycles must be present for feather-trace measurement parsing
-    if conf.PARAMS['cycles'] not in params:
-        params[conf.PARAMS['cycles']] = conf.DEFAULTS['cycles']
+    if PARAMS['cycles'] not in params:
+        params[PARAMS['cycles']] = DEFAULTS['cycles']
 
     return params
 
@@ -101,7 +102,7 @@ def parse_exp(exp_force):
     if not result:
         try:
             result = ExpPoint(exp.path)
-            cycles = exp.params[conf.PARAMS['cycles']]
+            cycles = exp.params[PARAMS['cycles']]
 
             # Write overheads into result
             ft.extract_ft_data(result, exp.path, exp.work_dir, cycles)
@@ -116,21 +117,31 @@ def parse_exp(exp_force):
 
     return (exp, result)
 
+def get_exps(args):
+    if args:
+        return args
+    elif os.path.exists(DEFAULTS['out-run']):
+        sys.stderr.write("Reading data from %s/*\n" % DEFAULTS['out-run'])
+        sched_dirs = os.listdir(DEFAULTS['out-run'])
+        return ['%s/%s' % (DEFAULTS['out-run'], d) for d in sched_dirs]
+    else:
+        sys.stderr.write("Reading data from current directory.\n")
+        return [os.getcwd()]
+
 def main():
     opts, args = parse_args()
-
-    args = args or [os.getcwd()]
+    exp_dirs  = get_exps(args)
 
     # Load exp parameters into a ColMap
     builder = ColMapBuilder()
-    exps = load_exps(args, builder, opts.force)
+    exps = load_exps(exp_dirs, builder, opts.force)
 
     # Don't track changes in ignored parameters
     if opts.ignore:
         for param in opts.ignore.split(","):
             builder.try_remove(param)
     # Always average multiple trials
-    builder.try_remove(conf.PARAMS['trial'])
+    builder.try_remove(PARAMS['trial'])
 
     col_map = builder.build()
     result_table = TupleTable(col_map)
@@ -175,7 +186,8 @@ def main():
         # No csvs to write, assume user meant to print out data
         if dir_map.is_empty():
             if not opts.verbose:
-                sys.stderr.write("Too little data to make csv files.\n")
+                sys.stderr.write("Too little data to make csv files, " +
+                                 "printing results.\n")
                 for key, exp in result_table:
                     for e in exp:
                         print(e)
diff --git a/plot_exps.py b/plot_exps.py
index 76e7396..15c54d0 100755
--- a/plot_exps.py
+++ b/plot_exps.py
@@ -6,7 +6,9 @@ import os
 import shutil as sh
 import sys
 import traceback
+
 from collections import namedtuple
+from config.config import DEFAULTS
 from multiprocessing import Pool, cpu_count
 from optparse import OptionParser
 from parse.col_map import ColMap,ColMapBuilder
@@ -17,7 +19,8 @@ def parse_args():
     parser = OptionParser("usage: %prog [options] [csv_dir]...")
 
     parser.add_option('-o', '--out-dir', dest='out_dir',
-                      help='directory for plot output', default='plot-data')
+                      help='directory for plot output',
+                      default=DEFAULTS['out-plot'])
     parser.add_option('-f', '--force', action='store_true', default=False,
                       dest='force', help='overwrite existing data')
     parser.add_option('-p', '--processors', default=max(cpu_count() - 1, 1),
@@ -139,21 +142,31 @@ def plot_dir(data_dir, out_dir, max_procs, force):
 
     sys.stderr.write('\n')
 
+def get_dirs(args):
+    if args:
+        return args
+    elif os.path.exists(DEFAULTS['out-parse']):
+        return [DEFAULTS['out-parse']]
+    else:
+        return os.getcwd()
+
 def main():
     opts, args = parse_args()
-    args = args or [os.getcwd()]
+    dirs = get_dirs(args)
 
     if opts.force and os.path.exists(opts.out_dir):
         sh.rmtree(opts.out_dir)
     if not os.path.exists(opts.out_dir):
         os.mkdir(opts.out_dir)
 
-    for dir in args:
-        if len(args) > 1:
+    for dir in dirs:
+        if len(dirs) > 1:
             out_dir = "%s/%s" % (opts.out_dir, os.path.split(dir)[1])
         else:
             out_dir = opts.out_dir
         plot_dir(dir, out_dir, opts.processors, opts.force)
 
+    sys.stderr.write("Plots saved in %s.\n" % opts.out_dir)
+
 if __name__ == '__main__':
     main()
diff --git a/run_exps.py b/run_exps.py
index 905d033..d7a06b5 100755
--- a/run_exps.py
+++ b/run_exps.py
@@ -2,13 +2,13 @@
 from __future__ import print_function
 
 import common as com
-import config.config as conf
 import os
 import re
 import shutil
 import sys
 import run.tracer as trace
 
+from config.config import PARAMS,DEFAULTS
 from collections import namedtuple
 from optparse import OptionParser
 from run.executable.executable import Executable
@@ -56,12 +56,13 @@ def parse_args():
     parser.add_option('-d', '--duration', dest='duration', type='int',
                       help='duration (seconds) of tasks')
     parser.add_option('-o', '--out-dir', dest='out_dir',
-                      help='directory for data output', default=("%s/run-data"%os.getcwd()))
+                      help='directory for data output',
+                      default=DEFAULTS['out-run'])
     parser.add_option('-p', '--params', dest='param_file',
                       help='file with experiment parameters')
     parser.add_option('-c', '--schedule-file', dest='sched_file',
                       help='name of schedule files within directories',
-                      default=conf.DEFAULTS['sched_file'])
+                      default=DEFAULTS['sched_file'])
     parser.add_option('-f', '--force', action='store_true', default=False,
                       dest='force', help='overwrite existing data')
     parser.add_option('-j', '--jabber', metavar='username@domain',
@@ -96,7 +97,7 @@ def convert_data(data):
             proc = (loc, match.group("CONTENT"))
             procs.append(proc)
         else:
-            prog = match.group("PROG") or conf.DEFAULTS['prog']
+            prog = match.group("PROG") or DEFAULTS['prog']
             spin = (prog, match.group("ARGS"))
             tasks.append(spin)
 
@@ -184,10 +185,10 @@ def verify_environment(exp_params):
 def run_parameter(exp_dir, out_dir, params, param_name):
     '''Run an executable (arguments optional) specified as a configurable
     @param_name in @params.'''
-    if conf.PARAMS[param_name] not in params:
+    if PARAMS[param_name] not in params:
         return
 
-    script_params = params[conf.PARAMS[param_name]]
+    script_params = params[PARAMS[param_name]]
 
     # Split into arguments and program name
     if type(script_params) != type([]):
@@ -210,22 +211,22 @@ def get_exp_params(cmd_scheduler, cmd_duration, file_params):
     '''Return ExpParam with configured values of all hardcoded params.'''
     kernel = copts = ""
 
-    scheduler = cmd_scheduler or file_params[conf.PARAMS['sched']]
-    duration  = cmd_duration  or file_params[conf.PARAMS['dur']] or\
-                conf.DEFAULTS['duration']
+    scheduler = cmd_scheduler or file_params[PARAMS['sched']]
+    duration  = cmd_duration  or file_params[PARAMS['dur']] or\
+                DEFAULTS['duration']
 
     # Experiments can specify required kernel name
-    if conf.PARAMS['kernel'] in file_params:
-        kernel = file_params[conf.PARAMS['kernel']]
+    if PARAMS['kernel'] in file_params:
+        kernel = file_params[PARAMS['kernel']]
 
     # Or required config options
-    if conf.PARAMS['copts'] in file_params:
-        copts = file_params[conf.PARAMS['copts']]
+    if PARAMS['copts'] in file_params:
+        copts = file_params[PARAMS['copts']]
 
     # Or required tracers
     requested = []
-    if conf.PARAMS['trace'] in file_params:
-        requested = file_params[conf.PARAMS['trace']]
+    if PARAMS['trace'] in file_params:
+        requested = file_params[PARAMS['trace']]
     tracers = trace.get_tracer_types(requested)
 
     # But only these two are mandatory
@@ -250,7 +251,7 @@ def load_experiment(sched_file, cmd_scheduler, cmd_duration,
 
     # Load parameter file
     param_file = param_file or \
-      "%s/%s" % (dir_name, conf.DEFAULTS['params_file'])
+      "%s/%s" % (dir_name, DEFAULTS['params_file'])
     if os.path.isfile(param_file):
         file_params = com.load_params(param_file)
     else:
@@ -277,19 +278,36 @@ def load_experiment(sched_file, cmd_scheduler, cmd_duration,
 
     # Save parameters used to run experiment in out_dir
     out_params = dict(file_params.items() +
-                      [(conf.PARAMS['sched'],  exp_params.scheduler),
-                       (conf.PARAMS['tasks'],  len(execs)),
-                       (conf.PARAMS['dur'],    exp_params.duration)])
+                      [(PARAMS['sched'],  exp_params.scheduler),
+                       (PARAMS['tasks'],  len(execs)),
+                       (PARAMS['dur'],    exp_params.duration)])
 
     # Feather-trace clock frequency saved for accurate overhead parsing
     ft_freq = com.ft_freq()
     if ft_freq:
-        out_params[conf.PARAMS['cycles']] = ft_freq
+        out_params[PARAMS['cycles']] = ft_freq
 
-    with open("%s/%s" % (out_dir, conf.DEFAULTS['params_file']), 'w') as f:
+    with open("%s/%s" % (out_dir, DEFAULTS['params_file']), 'w') as f:
         f.write(str(out_params))
 
 
+def get_exps(opts, args):
+    if args:
+        return args
+
+    # Default to sched_file > generated dirs
+    if os.path.exists(opts.sched_file):
+        sys.stderr.write("Reading schedule from %s.\n" % opts.sched_file)
+        return [opts.sched_file]
+    elif os.path.exists(DEFAULTS['out-gen']):
+        sys.stderr.write("Reading schedules from %s/*.\n" % DEFAULTS['out-gen'])
+        sched_dirs = os.listdir(DEFAULTS['out-gen'])
+        return ['%s/%s' % (DEFAULTS['out-gen'], d) for d in sched_dirs]
+    else:
+        sys.stderr.write("Run with -h to view options.\n");
+        sys.exit(1)
+
+
 def setup_jabber(target):
     try:
         from run.jabber import Jabber
@@ -300,6 +318,7 @@ def setup_jabber(target):
                          "Disabling instant messages.\n")
         return None
 
+
 def setup_email(target):
     try:
         from run.emailer import Emailer
@@ -313,6 +332,7 @@ def setup_email(target):
     sys.stderr.write(message + " Disabling email message.\n")
     return None
 
+
 def main():
     opts, args = parse_args()
 
@@ -321,7 +341,7 @@ def main():
     param_file = opts.param_file
     out_base   = os.path.abspath(opts.out_dir)
 
-    args = args or [opts.sched_file]
+    exps = get_exps(opts, args)
 
     created = False
     if not os.path.exists(out_base):
@@ -335,9 +355,9 @@ def main():
     invalid = 0
 
     jabber = setup_jabber(opts.jabber) if opts.jabber else None
-    email  = setup_email(opts.email) if opts.email else None
+    email  = setup_email(opts.email)   if opts.email  else None
 
-    for exp in args:
+    for exp in exps:
         path = "%s/%s" % (os.getcwd(), exp)
         out_dir = "%s/%s" % (out_base, os.path.split(exp.strip('/'))[1])
 
@@ -374,6 +394,7 @@ def main():
 
         ran += 1
 
+    # Clean out directory if it failed immediately
     if not os.listdir(out_base) and created and not succ:
         os.rmdir(out_base)
 
@@ -385,6 +406,10 @@ def main():
 
     print(message)
 
+    if succ:
+        sys.stderr.write("Successful experiment data saved in %s.\n" %
+                         opts.out_dir)
+
     if email:
         email.send(message)
         email.close()