From 4b75d2b98036195b78d339210ab8d7512d9f2164 Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Sat, 30 Mar 2013 11:28:51 -0400 Subject: Added option to ignore environment issues and rewrote part of the generation scripts for easier generator creation. --- common.py | 11 +++++---- gen/edf_generators.py | 30 ++++++++++--------------- gen/generator.py | 62 ++++++++++++++++++++++++++++++++++++++------------- gen_exps.py | 4 +++- run/proc_entry.py | 7 ++++-- run_exps.py | 28 ++++++++++++++++------- 6 files changed, 93 insertions(+), 49 deletions(-) diff --git a/common.py b/common.py index cba31bf..a26ac94 100644 --- a/common.py +++ b/common.py @@ -50,6 +50,12 @@ def get_config_option(option): else: raise IOError("No config file exists!") +def try_get_config_option(option, default): + try: + get_config_option(option) + except: + return default + def recordtype(typename, field_names, default=0): ''' Mutable namedtuple. Recipe from George Sakkis of MIT.''' field_names = tuple(map(str, field_names)) @@ -127,10 +133,7 @@ def load_params(fname): with open(fname, 'r') as f: data = f.read() try: - parsed = eval(data) - # Convert to defaultdict - for k in parsed: - params[k] = str(parsed[k]) + params = eval(data) except Exception as e: raise IOError("Invalid param file: %s\n%s" % (fname, e)) diff --git a/gen/edf_generators.py b/gen/edf_generators.py index c7267f7..9bf6b8f 100644 --- a/gen/edf_generators.py +++ b/gen/edf_generators.py @@ -1,15 +1,20 @@ import generator as gen import random -import schedcat.generator.tasks as tasks + +TP_TBASE = """#for $t in $task_set +{} $t.cost $t.period +#end for""" +TP_GLOB_TASK = TP_TBASE.format("") +TP_PART_TASK = TP_TBASE.format("-p $t.cpu") class EdfGenerator(gen.Generator): '''Creates sporadic task sets with the most common Litmus options.''' def __init__(self, name, templates, options, params): super(EdfGenerator, self).__init__(name, templates, - self.__make_options(params) + options, + self.__make_options() + options, params) - def __make_options(self, params): + def __make_options(self): '''Return generic EDF options.''' return [gen.Generator._dist_option('utils', ['uni-medium'], gen.NAMED_UTILIZATIONS, @@ -26,23 +31,12 @@ class EdfGenerator(gen.Generator): udist = self._create_dist('utilization', exp_params['utils'], gen.NAMED_UTILIZATIONS) - tg = tasks.TaskGenerator(period=pdist, util=udist) - ts = [] - tries = 0 - while len(ts) != exp_params['num_tasks'] and tries < 5: - ts = tg.make_task_set(max_tasks = exp_params['num_tasks']) - tries += 1 - if len(ts) != exp_params['num_tasks']: - print("Failed to create task set with parameters: %s" % exp_params) + ts = self._create_taskset(exp_params, pdist, udist) self._customize(ts, exp_params) - exp_params['task_set'] = ts - self._write_schedule(exp_params) - - del exp_params['task_set'] - del exp_params['num_tasks'] + self._write_schedule(dict(exp_params.items() + ('task_set', ts))) self._write_params(exp_params) def _customize(self, taskset, exp_params): @@ -53,7 +47,7 @@ class EdfGenerator(gen.Generator): class PartitionedGenerator(EdfGenerator): def __init__(self, name, templates, options, params): super(PartitionedGenerator, self).__init__(name, - templates + [gen.TP_PART_TASK], options, params) + templates + [TP_PART_TASK], options, params) def _customize(self, taskset, exp_params): start = 1 if exp_params['release_master'] else 0 @@ -78,5 +72,5 @@ class CedfGenerator(PartitionedGenerator): class GedfGenerator(EdfGenerator): def __init__(self, params={}): - super(GedfGenerator, self).__init__("GSN-EDF", [gen.TP_GLOB_TASK], + super(GedfGenerator, self).__init__("GSN-EDF", [TP_GLOB_TASK], [], params) diff --git a/gen/generator.py b/gen/generator.py index 8c3048b..3a6524d 100644 --- a/gen/generator.py +++ b/gen/generator.py @@ -1,10 +1,11 @@ import gen.rv as rv import os +import pprint +import schedcat.generator.tasks as tasks import shutil as sh from Cheetah.Template import Template -from collections import namedtuple -from common import get_config_option,num_cpus +from common import get_config_option,num_cpus,recordtype from config.config import DEFAULTS,PARAMS from gen.dp import DesignPointGenerator from parse.col_map import ColMapBuilder @@ -21,6 +22,7 @@ NAMED_UTILIZATIONS = { 'uni-light' : rv.uniform(0.001, 0.1), 'uni-medium' : rv.uniform( 0.1, 0.4), 'uni-heavy' : rv.uniform( 0.5, 0.9), + 'uni-mixed' : rv.uniform(0.001, .4), 'exp-light' : rv.exponential(0, 1, 0.10), 'exp-medium' : rv.exponential(0, 1, 0.25), @@ -36,15 +38,12 @@ NAMED_UTILIZATIONS = { '''Components of Cheetah template for schedule file''' TP_RM = """#if $release_master -release_master{1} +release_master{0} #end if""" -TP_TBASE = """#for $t in $task_set -{} $t.cost $t.period -#end for""" -TP_GLOB_TASK = TP_TBASE.format("") -TP_PART_TASK = TP_TBASE.format("-p $t.cpu") -GenOption = namedtuple('GenOption', ['name', 'types', 'default', 'help']) +GenOptionT = recordtype('GenOption', ['name', 'types', 'default', 'help', 'hidden']) +def GenOption(name, types, default, help, hidden = False): + return GenOptionT(name, types, default, help, hidden) class Generator(object): '''Creates all combinations @options specified by @params. @@ -92,36 +91,65 @@ class Generator(object): def _create_dist(self, name, value, named_dists): '''Attempt to create a distribution representing the data in @value. If @value is a string, use it as a key for @named_dists.''' - name = "%s distribution" % name # A list of values if type(value) == type([]): map(lambda x : self.__check_value(name, x, [float, int]), value) return rv.uniform_choice(value) elif type(value) in [float, int]: return lambda : value - elif value in named_dists: + elif named_dists and value in named_dists: return named_dists[value] else: raise ValueError("Invalid %s value: %s" % (name, value)) + def _create_taskset(self, params, periods, utils, max_util = None): + tg = tasks.TaskGenerator(period=periods, util=utils) + ts = [] + tries = 0 + while len(ts) != params['num_tasks'] and tries < 100: + ts = tg.make_task_set(max_tasks = params['num_tasks'], max_util=max_util) + tries += 1 + if len(ts) != params['num_tasks']: + print(("Only created task set of size %d < %d for params %s. " + + "Switching to light utilization.") % + (len(ts), params['num_tasks'], params)) + print("Switching to light util. This usually means the " + + "utilization distribution is too agressive.") + return self._create_taskset(params, periods, NAMED_UTILIZATIONS['uni-light'], + max_util) + return ts + def _write_schedule(self, params): '''Write schedule file using current template for @params.''' sched_file = self.out_dir + "/" + DEFAULTS['sched_file'] with open(sched_file, 'wa') as f: f.write(str(Template(self.template, searchList=[params]))) + def _write_params(self, params): '''Write out file with relevant parameters.''' + # Don't include this in the parameters. It will be automatically added + # in run_exps.py + if 'num_tasks' in params: + num_tasks = params.pop('num_tasks') + else: + num_tasks = 0 + exp_params_file = self.out_dir + "/" + DEFAULTS['params_file'] with open(exp_params_file, 'wa') as f: params['scheduler'] = self.name - f.write(str(params)) + pprint.pprint(params, f) + + if num_tasks: + params['num_tasks'] = num_tasks def __setup_params(self, params): '''Set default parameter values and check that values are valid.''' for option in self.options: if option.name not in params: params[option.name] = option.default + else: + option.hidden = True params[option.name] = self._check_value(option.name, option.types, params[option.name]) @@ -207,14 +235,16 @@ class Generator(object): if PARAMS['trial'] in dp: del dp[PARAMS['trial']] + HELP_INDENT = 17 def print_help(self): + display_options = [o for o in self.options if not o.hidden] s = str(Template("""Generator $name: #for $o in $options $o.name -- $o.help \tDefault: $o.default \tAllowed: $o.types - #end for""", searchList=vars(self))) + #end for""", searchList={'name':self.name, 'options':display_options})) # Has to be an easier way to print this out... for line in s.split("\n"): @@ -223,8 +253,8 @@ class Generator(object): for word in line.split(", "): i += len(word) res += [word] - if i > 80: + if i > 80 and len(word) < 80: print(", ".join(res[:-1])) - res = ["\t\t "+res[-1]] - i = line.index("'") + res = [" "*Generator.HELP_INDENT +res[-1]] + i = Generator.HELP_INDENT + len(word) print(", ".join(res)) diff --git a/gen_exps.py b/gen_exps.py index 20a6c6f..cb05fe7 100755 --- a/gen_exps.py +++ b/gen_exps.py @@ -6,12 +6,14 @@ import re import shutil as sh from gen.edf_generators import GedfGenerator,PedfGenerator,CedfGenerator +from gen.mc_generators import McGenerator from optparse import OptionParser # There has to be a better way to do this... GENERATORS = {'C-EDF':CedfGenerator, 'P-EDF':PedfGenerator, - 'G-EDF':GedfGenerator} + 'G-EDF':GedfGenerator, + 'MC':McGenerator} def parse_args(): parser = OptionParser("usage: %prog [options] [files...] " diff --git a/run/proc_entry.py b/run/proc_entry.py index 0b7f9ce..4ac2c51 100644 --- a/run/proc_entry.py +++ b/run/proc_entry.py @@ -8,5 +8,8 @@ class ProcEntry(object): def write_proc(self): if not os.path.exists(self.proc): raise Exception("Invalid proc entry %s" % self.proc) - with open(self.proc, 'w') as entry: - entry.write(self.data) + try: + with open(self.proc, 'w') as entry: + entry.write(self.data) + except: + print("Failed to write into %s value:\n%s" % (self.proc, self.data)) diff --git a/run_exps.py b/run_exps.py index 6873877..4a2d8ab 100755 --- a/run_exps.py +++ b/run_exps.py @@ -27,15 +27,24 @@ class InvalidConfig(Exception): self.results = results def __str__(self): - rstr = "'%s' - wanted: '%s', found: %s" - result = [rstr % (r.actual, r.param, r.wanted) for r in self.results] - return "Invalid kernel configuration\n" + result.join("\n") + rstr = "'%s'%swanted: '%s', found: %s" + messages = [] + for r in self.results: + # For pretty alignment + tabs = (3 - len(r.param)/8) + messages += [rstr % (r.param, '\t'*tabs, r.wanted, r.actual)] + + return "Invalid kernel configuration " +\ + "(ignore configuration with -i option).\n" + "\n".join(messages) def parse_args(): parser = OptionParser("usage: %prog [options] [sched_file]... [exp_dir]...") parser.add_option('-s', '--scheduler', dest='scheduler', help='scheduler for all experiments') + parser.add_option('-i', '--ignore-environment', dest='ignore', + action='store_true', default=False, + help='run experiments even in invalid environments ') parser.add_option('-d', '--duration', dest='duration', type='int', help='duration (seconds) of tasks') parser.add_option('-o', '--out-dir', dest='out_dir', @@ -87,7 +96,7 @@ def fix_paths(schedule, exp_dir, sched_file): if os.path.exists(abspath): args = args.replace(arg, abspath) break - elif re.match(r'.*\w+\.\w+', arg): + elif re.match(r'.*\w+\.[a-zA-Z]\w*', arg): print("WARNING: non-existent file '%s' may be referenced:\n\t%s" % (arg, sched_file)) @@ -108,9 +117,10 @@ def verify_environment(kernel, copts): results += [ConfigResult(param, wanted, actual)] if results: - raise InvalidKernel(results) + raise InvalidConfig(results) -def load_experiment(sched_file, scheduler, duration, param_file, out_dir): +def load_experiment(sched_file, scheduler, duration, + param_file, out_dir, ignore): if not os.path.isfile(sched_file): raise IOError("Cannot find schedule file: %s" % sched_file) @@ -146,7 +156,8 @@ def load_experiment(sched_file, scheduler, duration, param_file, out_dir): fix_paths(schedule, os.path.split(sched_file)[0], sched_file) - verify_environment(kernel, copts) + if not ignore: + verify_environment(kernel, copts) run_exp(exp_name, schedule, scheduler, kernel, duration, work_dir, out_dir) @@ -251,7 +262,8 @@ def main(): path = "%s/%s" % (path, opts.sched_file) try: - load_experiment(path, scheduler, duration, param_file, out_dir) + load_experiment(path, scheduler, duration, param_file, + out_dir, opts.ignore) succ += 1 except ExperimentDone: done += 1 -- cgit v1.2.2