From cb8db5d30ee769304c2c2b00f2a7d9bcb3c4098f Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Mon, 26 Nov 2012 16:02:48 -0500 Subject: Removed 2-step parse for scheduling statistics. --- parse_exps.py | 122 ++++++++++++++++------------------------------------------ 1 file changed, 34 insertions(+), 88 deletions(-) (limited to 'parse_exps.py') diff --git a/parse_exps.py b/parse_exps.py index d932b0d..c8cd8b1 100755 --- a/parse_exps.py +++ b/parse_exps.py @@ -2,11 +2,9 @@ from __future__ import print_function import config.config as conf -import copy import os import parse.ft as ft import parse.sched as st -import re import shutil as sh import sys @@ -22,13 +20,8 @@ def parse_args(): parser.add_option('-o', '--out', dest='out', help='file or directory for data output', default='parse-data') - - # TODO: this means nothing, also remove dests parser.add_option('-c', '--clean', action='store_true', default=False, dest='clean', help='do not output single-point csvs') - parser.add_option('-s', '--scale-against', dest='scale_against', - metavar='PARAM=VALUE', default="", - help='calculate task scaling factors against these configs') parser.add_option('-i', '--ignore', metavar='[PARAM...]', default="", help='ignore changing parameter values') parser.add_option('-f', '--force', action='store_true', default=False, @@ -41,136 +34,89 @@ def parse_args(): return parser.parse_args() -ExpData = namedtuple('ExpData', ['name', 'params', 'data_files', 'is_base']) -DataFiles = namedtuple('DataFiles', ['st']) +ExpData = namedtuple('ExpData', ['path', 'params', 'work_dir']) def get_exp_params(data_dir, col_map): param_file = "%s/%s" % (data_dir, conf.DEFAULTS['params_file']) if not os.path.isfile: raise Exception("No param file '%s' exists!" % param_file) - # Keep only params that uniquely identify the experiment + # Ignore 'magic' parameters used by these scripts params = load_params(param_file) for ignored in conf.PARAMS.itervalues(): - # Always include cycles or overhead parsing fails + # With the exception of cycles which is used by overhead parsing if ignored in params and ignored != conf.PARAMS['cycles']: params.pop(ignored) - # Track all changed params + # Store parameters in col_map, which will track which parameters change + # across experiments for key, value in params.iteritems(): col_map.try_add(key, value) + # Cycles must be present if conf.PARAMS['cycles'] not in params: params[conf.PARAMS['cycles']] = conf.DEFAULTS['cycles'] return params -def gen_exp_data(exp_dirs, base_conf, col_map, force): - plain_exps = [] - scaling_bases = [] +def load_exps(exp_dirs, col_map, clean): + exps = [] - sys.stderr.write("Generating data...\n") + sys.stderr.write("Loading experiments...\n") - for i, data_dir in enumerate(exp_dirs): + for data_dir in exp_dirs: if not os.path.isdir(data_dir): raise IOError("Invalid experiment '%s'" % os.path.abspath(data_dir)) - tmp_dir = data_dir + "/tmp" - if not os.path.exists(tmp_dir): - os.mkdir(tmp_dir) - - # Read and translate exp output files - params = get_exp_params(data_dir, col_map) - st_output = st.get_st_output(data_dir, tmp_dir, force) - - if base_conf and base_conf.viewitems() & params.viewitems(): - if not st_output: - raise Exception("Scaling base '%s' useless without sched data!" - % data_dir) - is_base = True - - base_params = copy.deepcopy(params) - base_params.pop(base_conf.keys()[0]) + # Used to store error output and debugging info + work_dir = data_dir + "/tmp" - base_exp = ExpData(data_dir, base_params, - DataFiles(st_output), True) - scaling_bases += [base_exp] - else: - is_base = False + if not os.path.exists(work_dir): + os.mkdir(work_dir) + elif clean: + sh.rmtree(work_dir) - # Create experiment named after the data dir - exp_data = ExpData(data_dir, params, - DataFiles(st_output), is_base) + params = get_exp_params(data_dir, col_map) - plain_exps += [exp_data] + exps += [ ExpData(data_dir, params, work_dir) ] - sys.stderr.write('\r {0:.2%}'.format(float(i)/len(exp_dirs))) - sys.stderr.write('\n') - return (plain_exps, scaling_bases) + return exps def main(): opts, args = parse_args() args = args or [os.getcwd()] - # Configuration key for task systems used to calculate task - # execution scaling factors - base_conf = dict(re.findall("(.*)=(.*)", opts.scale_against)) - + # Load exp parameters into col_map col_map = ColMap() + exps = load_exps(args, col_map, opts.force) - (plain_exps, scaling_bases) = gen_exp_data(args, base_conf, col_map, opts.force) - - if base_conf and base_conf.keys()[0] not in col_map: - raise IOError("Base column '%s' not present in any parameters!" % - base_conf.keys()[0]) - - base_map = copy.deepcopy(col_map) + # Don't track changes in ignored parameters if opts.ignore: for param in opts.ignore.split(","): col_map.try_remove(param) - base_table = TupleTable(base_map) # For tracking 'base' experiments - result_table = TupleTable(col_map) # For generating output - - # Used to find matching scaling_base for each experiment - for base in scaling_bases: - base_table.add_exp(base.params, base) + result_table = TupleTable(col_map) sys.stderr.write("Parsing data...\n") - for exp in args: - result = ExpPoint(exp) - params = get_exp_params(exp, col_map) - # Write overheads into result - ft.extract_ft_data(result, exp, - params[conf.PARAMS['cycles']], - exp + "/tmp") - - if opts.verbose: - print(result) - - for i,exp in enumerate(plain_exps): - result = ExpPoint(exp.name) - - if exp.data_files.st: - base = None - if base_conf and not exp.is_base: - # Try to find a scaling base - base_params = copy.deepcopy(exp.params) - base_params.pop(base_conf.keys()[0]) - base = base_table.get_exps(base_params)[0] + for i,exp in enumerate(exps): + result = ExpPoint(exp.path) + cycles = exp.params[conf.PARAMS['cycles']] - # Write deadline misses / tardiness into result - st.extract_sched_data(exp.data_files.st, result, - base.data_files.st if base else None) + # Write overheads into result + ft.extract_ft_data(result, exp.path, exp.work_dir, cycles) - result_table.add_exp(exp.params, result) + # Write scheduling statistics into result + st.extract_sched_data(result, exp.path, exp.work_dir) if opts.verbose: print(result) else: - sys.stderr.write('\r {0:.2%}'.format(float(i)/len(plain_exps))) + sys.stderr.write('\r {0:.2%}'.format(float(i)/len(exps))) + + result_table.add_exp(exp.params, result) + sys.stderr.write('\n') if opts.force and os.path.exists(opts.out): -- cgit v1.2.2