From 94cc65997d237ddeab24d396f06bb93bc0644a9d Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Tue, 30 Apr 2013 11:44:37 -0400 Subject: Cleaned up run_exps.py parameters. --- README.md | 6 ++---- config/config.py | 18 ++++++++-------- gen/generator.py | 6 +++--- parse_exps.py | 4 ++-- run_exps.py | 62 ++++++++++++++++++++++++++++++-------------------------- 5 files changed, 49 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index b074aa5..1f38978 100644 --- a/README.md +++ b/README.md @@ -156,16 +156,14 @@ You can specify your own spin programs to run as well instead of rtspin by putti $ echo "colorspin -f color1.csv 10 20" > test.sched ``` -You can specify parameters for an experiment in a file instead of on the command line using params.py (the `-p` option lets you choose the name of this file if `params.py` is not for you): +You can specify parameters for an experiment in a file instead of on the command line using params.py: ```bash $ echo "{'scheduler':'GSN-EDF', 'duration':10}" > params.py $ run_exps.py test.sched ``` -You can also run multiple experiments with a single command, provided a directory with a schedule file exists for each. By default, the program will look for sched.py for the schedule file and params.py for the parameter file, but this behavior can be changed using the `-p` and `-c` options. - -You can include non-relevant parameters which `run_exps.py` does not understand in `params.py`. These parameters will be saved with the data output by `run_exps.py`. This is useful for tracking variations in system parameters versus experimental results. In the following example, multiple experiments are demonstrated and an extra parameter `test-param` is included: +You can also run multiple experiments with a single command, provided a directory with a schedule file exists for each. You can include non-relevant parameters which `run_exps.py` does not understand in `params.py`. These parameters will be saved with the data output by `run_exps.py`. This is useful for tracking variations in system parameters versus experimental results. In the following example, multiple experiments are demonstrated and an extra parameter `test-param` is included: ```bash $ mkdir test1 diff --git a/config/config.py b/config/config.py index 5e6f9e3..cfdfb05 100644 --- a/config/config.py +++ b/config/config.py @@ -14,12 +14,14 @@ BINS = {'rtspin' : get_executable_hint('rtspin', 'liblitmus'), # Optional, as sched_trace is not a publically supported repository 'st_show' : get_executable_hint('st_show', 'sched_trace', True)} -'''Names of output files.''' -FILES = {'ft_data' : 'ft.bin', - 'ft_matches' : r'(ft.*\.bin$)|(.*\.ft)', - 'linux_data' : 'trace.dat', - 'sched_data' : 'st-{}.bin', - 'log_data' : 'trace.slog'} +'''Names of data files.''' +FILES = {'params_file' : 'params.py', + 'sched_file' : 'sched.py', + 'ft_data' : 'ft.bin', + 'ft_matches' : r'(ft.*\.bin$)|(.*\.ft)', + 'linux_data' : 'trace.dat', + 'sched_data' : 'st-{}.bin', + 'log_data' : 'trace.slog'} '''Default parameter names in params.py.''' PARAMS = {'sched' : 'scheduler', # Scheduler used by run_exps @@ -35,9 +37,7 @@ PARAMS = {'sched' : 'scheduler', # Scheduler used by run_exps } '''Default values for program options.''' -DEFAULTS = {'params_file' : 'params.py', - 'sched_file' : 'sched.py', - 'duration' : 10, +DEFAULTS = {'duration' : 10, 'prog' : 'rtspin', 'out-gen' : 'exps', 'out-run' : 'run-data', diff --git a/gen/generator.py b/gen/generator.py index 6a07616..0999e84 100644 --- a/gen/generator.py +++ b/gen/generator.py @@ -6,7 +6,7 @@ import shutil as sh from Cheetah.Template import Template from common import get_config_option,num_cpus,recordtype -from config.config import DEFAULTS,PARAMS +from config.config import FILES,PARAMS from gen.dp import DesignPointGenerator from parse.col_map import ColMapBuilder @@ -121,7 +121,7 @@ class Generator(object): def _write_schedule(self, params): '''Write schedule file using current template for @params.''' - sched_file = self.out_dir + "/" + DEFAULTS['sched_file'] + sched_file = self.out_dir + "/" + FILES['sched_file'] with open(sched_file, 'wa') as f: f.write(str(Template(self.template, searchList=[params]))) @@ -135,7 +135,7 @@ class Generator(object): else: tasks = 0 - exp_params_file = self.out_dir + "/" + DEFAULTS['params_file'] + exp_params_file = self.out_dir + "/" + FILES['params_file'] with open(exp_params_file, 'wa') as f: params['scheduler'] = self.scheduler pprint.pprint(params, f) diff --git a/parse_exps.py b/parse_exps.py index cc4372a..94c30a4 100755 --- a/parse_exps.py +++ b/parse_exps.py @@ -12,7 +12,7 @@ import sys import traceback from collections import namedtuple -from config.config import DEFAULTS,PARAMS +from config.config import FILES,DEFAULTS,PARAMS from optparse import OptionParser from parse.point import ExpPoint from parse.tuple_table import TupleTable @@ -85,7 +85,7 @@ def parse_exp(exp_force): def get_exp_params(data_dir, cm_builder): - param_file = "%s/%s" % (data_dir, DEFAULTS['params_file']) + param_file = "%s/%s" % (data_dir, FILES['params_file']) if os.path.isfile(param_file): params = com.load_params(param_file) diff --git a/run_exps.py b/run_exps.py index 1d46b45..1bad2a3 100755 --- a/run_exps.py +++ b/run_exps.py @@ -9,9 +9,9 @@ import shutil import sys import run.tracer as trace -from config.config import PARAMS,DEFAULTS +from config.config import PARAMS,DEFAULTS,FILES from collections import namedtuple -from optparse import OptionParser +from optparse import OptionParser,OptionGroup from parse.enum import Enum from run.executable.executable import Executable from run.experiment import Experiment,ExperimentDone,SystemCorrupted @@ -61,30 +61,34 @@ def parse_args(): parser.add_option('-s', '--scheduler', dest='scheduler', help='scheduler for all experiments') + parser.add_option('-d', '--duration', dest='duration', type='int', + help='duration (seconds) of tasks') parser.add_option('-i', '--ignore-environment', dest='ignore', action='store_true', default=False, help='run experiments even in invalid environments ') - parser.add_option('-d', '--duration', dest='duration', type='int', - help='duration (seconds) of tasks') + parser.add_option('-f', '--force', action='store_true', default=False, + dest='force', help='overwrite existing data') parser.add_option('-o', '--out-dir', dest='out_dir', help='directory for data output', default=DEFAULTS['out-run']) - parser.add_option('-p', '--params', dest='param_file', - help='file with experiment parameters') - parser.add_option('-c', '--schedule-file', dest='sched_file', - help='name of schedule files within directories', - default=DEFAULTS['sched_file']) - parser.add_option('-f', '--force', action='store_true', default=False, - dest='force', help='overwrite existing data') - parser.add_option('-j', '--jabber', metavar='username@domain', - dest='jabber', default=None, - help='send a jabber message when an experiment completes') - parser.add_option('-e', '--email', metavar='username@server', - dest='email', default=None, - help='send an email when all experiments complete') - parser.add_option('-r', '--retry', dest='retry', - action='store_true', default=False, - help='retry failed experiments') + + group = OptionGroup(parser, "Communication Options") + group.add_option('-j', '--jabber', metavar='username@domain', + dest='jabber', default=None, + help='send a jabber message when an experiment completes') + group.add_option('-e', '--email', metavar='username@server', + dest='email', default=None, + help='send an email when all experiments complete') + parser.add_option_group(group) + + group = OptionGroup(parser, "Persistence Options") + group.add_option('-r', '--retry', dest='retry', action='store_true', + default=False, help='retry failed experiments') + group.add_option('-c', '--crontab', dest='crontab', + action='store_true', default=False, + help='use crontab to resume interrupted script after ' + 'system restarts. implies --retry') + parser.add_option_group(group) return parser.parse_args() @@ -220,12 +224,12 @@ def run_script(script_params, exp, exp_dir, out_dir): out.close() -def make_exp_params(cmd_scheduler, cmd_duration, sched_dir, param_file): +def make_exp_params(cmd_scheduler, cmd_duration, sched_dir): '''Return ExpParam with configured values of all hardcoded params.''' kernel = copts = "" # Load parameter file - param_file = param_file or "%s/%s" % (sched_dir, DEFAULTS['params_file']) + param_file = "%s/%s" % (sched_dir, FILES['params_file']) if os.path.isfile(param_file): fparams = com.load_params(param_file) else: @@ -303,7 +307,7 @@ def run_experiment(data, start_message, ignore, jabber): if ft_freq: out_params[PARAMS['cycles']] = ft_freq - out_param_f = "%s/%s" % (data.out_dir, DEFAULTS['params_file']) + out_param_f = "%s/%s" % (data.out_dir, FILES['params_file']) with open(out_param_f, 'w') as f: pprint.pprint(out_params, f) @@ -320,7 +324,7 @@ def make_paths(exp, opts, out_base_dir): shutil.rmtree(out_dir) if os.path.isdir(path): - sched_file = "%s/%s" % (path, opts.sched_file) + sched_file = "%s/%s" % (path, FILES['sched_file']) else: sched_file = path @@ -348,10 +352,10 @@ def get_exps(opts, args, out_base_dir): '''Return list of ExpDatas''' if not args: - if os.path.exists(opts.sched_file): + if os.path.exists(FILES['sched_file']): # Default to sched_file in current directory - sys.stderr.write("Reading schedule from %s.\n" % opts.sched_file) - args = [opts.sched_file] + sys.stderr.write("Reading schedule from %s.\n" % FILES['sched_file']) + args = [FILES['sched_file']] elif os.path.exists(DEFAULTS['out-gen']): # Then try experiments created by gen_exps sys.stderr.write("Reading schedules from %s/*.\n" % DEFAULTS['out-gen']) @@ -370,8 +374,8 @@ def get_exps(opts, args, out_base_dir): name = path[len(common):] sched_dir = os.path.split(sched_file)[0] - exp_params = make_exp_params(opts.scheduler, opts.duration, - sched_dir, opts.param_file) + + exp_params = make_exp_params(opts.scheduler, opts.duration, sched_dir) exps += [ExpData(name, exp_params, sched_file, out_dir, 0, ExpState.None)] -- cgit v1.2.2