import gen.rv as rv import os import pprint import schedcat.generator.tasks as tasks import shutil as sh from Cheetah.Template import Template from common import get_config_option,num_cpus,recordtype from config.config import FILES,PARAMS from gen.dp import DesignPointGenerator from parse.col_map import ColMapBuilder NAMED_PERIODS = { 'harmonic' : rv.uniform_choice([25, 50, 100, 200]), 'uni-short' : rv.uniform_int( 3, 33), 'uni-moderate' : rv.uniform_int(10, 100), 'uni-long' : rv.uniform_int(50, 250), } NAMED_UTILIZATIONS = { 'uni-very-light': rv.uniform(0.0001, 0.001), 'uni-light' : rv.uniform(0.001, 0.1), 'uni-medium' : rv.uniform( 0.1, 0.4), 'uni-heavy' : rv.uniform( 0.5, 0.9), 'uni-mixed' : rv.uniform(0.001, .4), 'exp-light' : rv.exponential(0, 1, 0.10), 'exp-medium' : rv.exponential(0, 1, 0.25), 'exp-heavy' : rv.exponential(0, 1, 0.50), 'bimo-light' : rv.multimodal([(rv.uniform(0.001, 0.5), 8), (rv.uniform( 0.5, 0.9), 1)]), 'bimo-medium' : rv.multimodal([(rv.uniform(0.001, 0.5), 6), (rv.uniform( 0.5, 0.9), 3)]), 'bimo-heavy' : rv.multimodal([(rv.uniform(0.001, 0.5), 4), (rv.uniform( 0.5, 0.9), 5)]), } '''Components of Cheetah template for schedule file''' TP_RM = """#if $release_master release_master{0} #end if""" GenOptionT = recordtype('GenOption', ['name', 'types', 'default', 'help', 'hidden']) def GenOption(name, types, default, help, hidden = False): return GenOptionT(name, types, default, help, hidden) class Generator(object): '''Creates all combinations @options specified by @params. This class also performs checks of parameter values and prints out help. All subclasses must implement _create_exp. ''' def __init__(self, scheduler, templates, options, params): self.options = self.__make_options(params) + options self.__setup_params(params) self.params = params self.template = "\n".join([TP_RM] + templates) self.scheduler = scheduler def __make_options(self, params): '''Return generic Litmus options.''' # Guess defaults using the properties of this computer if 'cpus' in params: cpus = min(map(int, params['cpus'])) else: cpus = num_cpus() try: rm_config = get_config_option("RELEASE_MASTER") and True except: rm_config = False release_master = list(set([False, bool(rm_config)])) return [GenOption('tasks', int, range(cpus, 10*cpus, cpus), 'Number of tasks per experiment.'), GenOption('cpus', int, [cpus], 'Number of processors on target system.'), GenOption('release_master', [True,False], release_master, 'Redirect release interrupts to a single CPU.'), #GenOption('post', str, ['refill.sh'], 'Post script.'), GenOption('duration', float, [10], 'Experiment duration.')] @staticmethod def _dist_option(name, default, distribution, help): return GenOption(name, [str, float, type([])] + distribution.keys(), default, help) def _create_dist(self, name, value, named_dists): '''Attempt to create a distribution representing the data in @value. If @value is a string, use it as a key for @named_dists.''' # A list of values if type(value) == type([]): map(lambda x : self.__check_value(name, x, [float, int]), value) return rv.uniform_choice(value) elif type(value) in [float, int]: return lambda : value elif named_dists and value in named_dists: return named_dists[value] else: raise ValueError("Invalid %s value: %s" % (name, value)) def _create_taskset(self, params, periods, utils, max_util = None): tg = tasks.TaskGenerator(period=periods, util=utils) ts = [] tries = 0 while len(ts) != params['tasks'] and tries < 100: ts = tg.make_task_set(max_tasks = params['tasks'], max_util=max_util) tries += 1 if len(ts) != params['tasks']: print(("Only created task set of size %d < %d for params %s. " + "Switching to light utilization.") % (len(ts), params['tasks'], params)) print("Switching to very-light util. This usually means the " + "utilization distribution is too agressive.") return self._create_taskset(params, periods, NAMED_UTILIZATIONS['uni-very-light'], max_util) return ts def _write_schedule(self, params): '''Write schedule file using current template for @params.''' sched_file = self.out_dir + "/" + FILES['sched_file'] with open(sched_file, 'wa') as f: f.write(str(Template(self.template, searchList=[params]))) def _write_params(self, params): '''Write out file with relevant parameters.''' # Don't include this in the parameters. It will be automatically added # in run_exps.py if 'tasks' in params: tasks = params.pop('tasks') else: tasks = 0 exp_params_file = self.out_dir + "/" + FILES['params_file'] with open(exp_params_file, 'wa') as f: params['scheduler'] = self.scheduler pprint.pprint(params, f) if tasks: params['tasks'] = tasks def __setup_params(self, params): '''Set default parameter values and check that values are valid.''' for option in self.options: if option.name not in params: val = option.default val = val if type(val) == type([]) else [val] params[option.name] = val else: option.hidden = True params[option.name] = self._check_value(option.name, option.types, params[option.name]) return params def _check_value(self, name, types, val): '''Raise an exception if the value of type of @val is not specified in @types. Returns a copy of @val with strings converted to raw Python types, if possible.''' if types == float: types = [float, int] if type(types) != type([]): types = [types] if type(val) != type([]): val = [val] retval = [] for v in val: # Has to be a better way to find this v = False if v in ['f', 'False', 'false', 'n', 'no'] else v v = True if v in ['t', 'True', 'true', 'y', 'yes'] else v if type(v) not in types and v not in types: # Try and convert v to one of the specified types parsed = None for t in types: try: parsed = t(v) break except: pass if parsed: retval += [parsed] else: raise TypeError("Invalid %s value: '%s'" % (name, v)) else: retval += [v] return retval def _create_exp(self, exp_params, out_dir): '''Overridden by subclasses.''' raise NotImplementedError def create_exps(self, out_dir, force, trials): '''Create experiments for all possible combinations of params in @out_dir. Overwrite existing files if @force is True.''' builder = ColMapBuilder() # Track changing values so only relevant parameters are included # in directory names for dp in DesignPointGenerator(self.params): for k, v in dp.iteritems(): builder.try_add(k, v) col_map = builder.build() for dp in DesignPointGenerator(self.params): for trial in xrange(trials): # Create directory name from relevant parameters dir_leaf = "sched=%s_%s" % (self.scheduler, col_map.encode(dp)) dir_leaf = dir_leaf.strip('_') # If there are none dir_leaf += ("_trial=%s" % trial) if trials > 1 else "" dir_path = "%s/%s" % (out_dir, dir_leaf.strip('_')) if os.path.exists(dir_path): if force: sh.rmtree(dir_path) else: print("Skipping existing experiment: '%s'" % dir_path) continue os.mkdir(dir_path) if trials > 1: dp[PARAMS['trial']] = trial self.out_dir = dir_path self._create_exp(dict(dp)) del(self.out_dir) if PARAMS['trial'] in dp: del dp[PARAMS['trial']] HELP_INDENT = 17 def print_help(self): display_options = [o for o in self.options if not o.hidden] s = str(Template("""scheduler $scheduler: #for $o in $options $o.name -- $o.help \tDefault: $o.default \tAllowed: $o.types #end for""", searchList={'scheduler':self.scheduler, 'options':display_options})) # Has to be an easier way to print this out... for line in s.split("\n"): res = [] i = 0 for word in line.split(", "): i += len(word) res += [word] if i > 80 and len(word) < 80: print(", ".join(res[:-1])) res = [" "*Generator.HELP_INDENT +res[-1]] i = Generator.HELP_INDENT + len(word) print(", ".join(res)) generators = {} def register_generator(name, clazz): generators[name] = clazz def get_generators(): return generators