Scripts read directories created by other scripts if no arguments.

With no arguments, all scripts first try to load the current directory. If the
current directory has no data, the scripts search for the output of the previous
scripts in the toolchain, e.g. parse_exps.py loads run-data/*, created by
run_exps.py.

This commit also switched messages to stderr where they belong, and adds in
missing lock and unlock overheads.

5 files changed, 102 insertions, 44 deletions

diff --git a/config/config.py b/config/config.py
index cbac6b2..1ac468b 100644
--- a/config/config.py
+++ b/config/config.py
@@ -39,13 +39,18 @@ DEFAULTS = {'params_file' : 'params.py',
             'sched_file'  : 'sched.py',
             'duration'    : 10,
             'prog'        : 'rtspin',
+            'out-gen'     : 'exps',
+            'out-run'     : 'run-data',
+            'out-parse'   : 'parse-data',
+            'out-plot'    : 'plot-data',
             'cycles'      : ft_freq() or 2000}
 
+
 '''Default sched_trace events (this is all of them).'''
 SCHED_EVENTS = range(501, 513)
 
 '''Overhead events.'''
-OVH_BASE_EVENTS  = ['SCHED', 'RELEASE', 'SCHED2', 'TICK', 'CXS']
+OVH_BASE_EVENTS  = ['SCHED', 'RELEASE', 'SCHED2', 'TICK', 'CXS', 'LOCK', 'UNLOCK']
 OVH_ALL_EVENTS   = ["%s_%s" % (e, t) for (e,t) in
                     itertools.product(OVH_BASE_EVENTS, ["START","END"])]
 OVH_ALL_EVENTS  += ['RELEASE_LATENCY']
diff --git a/gen_exps.py b/gen_exps.py
index 6488cdc..b847661 100755
--- a/gen_exps.py
+++ b/gen_exps.py
@@ -7,6 +7,7 @@ import re
 import shutil as sh
 import sys
 
+from config.config import DEFAULTS
 from optparse import OptionParser
 
 def parse_args():
@@ -15,7 +16,7 @@ def parse_args():
 
     parser.add_option('-o', '--out-dir', dest='out_dir',
                       help='directory for data output',
-                      default=("%s/exps"%os.getcwd()))
+                      default=("%s/%s"% (os.getcwd(), DEFAULTS['out-gen'])))
     parser.add_option('-f', '--force', action='store_true', default=False,
                       dest='force', help='overwrite existing data')
     parser.add_option('-n', '--num-trials', default=1, type='int', dest='trials',
@@ -51,9 +52,9 @@ def main():
     if opts.described != None:
         for generator in opts.described.split(','):
             if generator not in gen.get_generators():
-                print("No generator '%s'" % generator)
+                sys.stderr.write("No generator '%s'\n" % generator)
             else:
-                sys.stdout.write("Generator '%s', " % generator)
+                print("Generator '%s', " % generator)
                 gen.get_generators()[generator]().print_help()
     if opts.list_gens or opts.described:
         return 0
@@ -85,7 +86,7 @@ def main():
         if gen_name not in gen.get_generators():
             raise ValueError("Invalid generator '%s'" % gen_name)
 
-        print("Creating experiments using %s generator..." % gen_name)
+        sys.stderr.write("Creating experiments with %s generator...\n" % gen_name)
 
         params = dict(gen_params.items() + global_params.items())
         clazz  = gen.get_generators()[gen_name]
@@ -94,5 +95,7 @@ def main():
 
         generator.create_exps(opts.out_dir, opts.force, opts.trials)
 
+    sys.stderr.write("Experiments saved in %s.\n" % opts.out_dir)
+
 if __name__ == '__main__':
     main()
diff --git a/parse_exps.py b/parse_exps.py
index c254536..d07378c 100755
--- a/parse_exps.py
+++ b/parse_exps.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 from __future__ import print_function
 
-import config.config as conf
 import os
 import parse.ft as ft
 import parse.sched as st
@@ -12,6 +11,7 @@ import traceback
 
 from collections import namedtuple
 from common import load_params
+from config.config import DEFAULTS,PARAMS
 from optparse import OptionParser
 from parse.point import ExpPoint
 from parse.tuple_table import TupleTable
@@ -22,7 +22,8 @@ def parse_args():
     parser = OptionParser("usage: %prog [options] [data_dir]...")
 
     parser.add_option('-o', '--out', dest='out',
-                      help='file or directory for data output', default='parse-data')
+                      help='file or directory for data output',
+                      default=DEFAULTS['out-parse'])
     parser.add_option('-i', '--ignore', metavar='[PARAM...]', default="",
                       help='ignore changing parameter values')
     parser.add_option('-f', '--force', action='store_true', default=False,
@@ -41,7 +42,7 @@ def parse_args():
 ExpData = namedtuple('ExpData', ['path', 'params', 'work_dir'])
 
 def get_exp_params(data_dir, cm_builder):
-    param_file = "%s/%s" % (data_dir, conf.DEFAULTS['params_file'])
+    param_file = "%s/%s" % (data_dir, DEFAULTS['params_file'])
     if os.path.isfile(param_file):
         params = load_params(param_file)
 
@@ -53,8 +54,8 @@ def get_exp_params(data_dir, cm_builder):
          params = {}
 
     # Cycles must be present for feather-trace measurement parsing
-    if conf.PARAMS['cycles'] not in params:
-        params[conf.PARAMS['cycles']] = conf.DEFAULTS['cycles']
+    if PARAMS['cycles'] not in params:
+        params[PARAMS['cycles']] = DEFAULTS['cycles']
 
     return params
 
@@ -101,7 +102,7 @@ def parse_exp(exp_force):
     if not result:
         try:
             result = ExpPoint(exp.path)
-            cycles = exp.params[conf.PARAMS['cycles']]
+            cycles = exp.params[PARAMS['cycles']]
 
             # Write overheads into result
             ft.extract_ft_data(result, exp.path, exp.work_dir, cycles)
@@ -116,21 +117,31 @@ def parse_exp(exp_force):
 
     return (exp, result)
 
+def get_exps(args):
+    if args:
+        return args
+    elif os.path.exists(DEFAULTS['out-run']):
+        sys.stderr.write("Reading data from %s/*\n" % DEFAULTS['out-run'])
+        sched_dirs = os.listdir(DEFAULTS['out-run'])
+        return ['%s/%s' % (DEFAULTS['out-run'], d) for d in sched_dirs]
+    else:
+        sys.stderr.write("Reading data from current directory.\n")
+        return [os.getcwd()]
+
 def main():
     opts, args = parse_args()
-
-    args = args or [os.getcwd()]
+    exp_dirs  = get_exps(args)
 
     # Load exp parameters into a ColMap
     builder = ColMapBuilder()
-    exps = load_exps(args, builder, opts.force)
+    exps = load_exps(exp_dirs, builder, opts.force)
 
     # Don't track changes in ignored parameters
     if opts.ignore:
         for param in opts.ignore.split(","):
             builder.try_remove(param)
     # Always average multiple trials
-    builder.try_remove(conf.PARAMS['trial'])
+    builder.try_remove(PARAMS['trial'])
 
     col_map = builder.build()
     result_table = TupleTable(col_map)
@@ -175,7 +186,8 @@ def main():
         # No csvs to write, assume user meant to print out data
         if dir_map.is_empty():
             if not opts.verbose:
-                sys.stderr.write("Too little data to make csv files.\n")
+                sys.stderr.write("Too little data to make csv files, " +
+                                 "printing results.\n")