diff options
| author | Jonathan Herman <hermanjl@cs.unc.edu> | 2013-02-07 11:21:23 -0500 |
|---|---|---|
| committer | Jonathan Herman <hermanjl@cs.unc.edu> | 2013-02-07 11:21:23 -0500 |
| commit | 39020cf5ae3030bc15035925a0c72eb44eea67b7 (patch) | |
| tree | fb82b339c1e5c14334f8d9839f8d836b120fbb08 | |
| parent | d524da9bd072ad1be4ac0d633e3e783094ddc2d7 (diff) | |
Added gen_exps.py script.
| -rw-r--r-- | common.py | 26 | ||||
| -rw-r--r-- | config/config.py | 16 | ||||
| -rw-r--r-- | gen/__init__.py | 0 | ||||
| -rw-r--r-- | gen/dp.py | 33 | ||||
| -rw-r--r-- | gen/generators.py | 257 | ||||
| -rw-r--r-- | gen/rv.py | 86 | ||||
| -rwxr-xr-x[-rw-r--r--] | gen_exps.py | 98 | ||||
| -rw-r--r-- | parse/ft.py | 4 | ||||
| -rw-r--r-- | parse/sched.py | 2 | ||||
| -rw-r--r-- | parse/tuple_table.py | 9 | ||||
| -rwxr-xr-x | parse_exps.py | 13 | ||||
| -rwxr-xr-x | run_exps.py | 1 |
12 files changed, 525 insertions, 20 deletions
| @@ -1,9 +1,14 @@ | |||
| 1 | import os | ||
| 2 | import re | ||
| 3 | import subprocess | ||
| 1 | import sys | 4 | import sys |
| 5 | |||
| 2 | from collections import defaultdict | 6 | from collections import defaultdict |
| 3 | from textwrap import dedent | 7 | from textwrap import dedent |
| 4 | 8 | ||
| 5 | def get_executable(prog, hint, optional=False): | 9 | def get_executable(prog, hint, optional=False): |
| 6 | import os | 10 | '''Search for @prog in system PATH. Print @hint if no binary is found.''' |
| 11 | |||
| 7 | def is_exe(fpath): | 12 | def is_exe(fpath): |
| 8 | return os.path.isfile(fpath) and os.access(fpath, os.X_OK) | 13 | return os.path.isfile(fpath) and os.access(fpath, os.X_OK) |
| 9 | 14 | ||
| @@ -19,12 +24,29 @@ def get_executable(prog, hint, optional=False): | |||
| 19 | 24 | ||
| 20 | if not optional: | 25 | if not optional: |
| 21 | sys.stderr.write("Cannot find executable '%s' in PATH. This is a part " | 26 | sys.stderr.write("Cannot find executable '%s' in PATH. This is a part " |
| 22 | "of '%s' which should be added to PATH to run." % | 27 | "of '%s' which should be added to PATH to run.\n" % |
| 23 | (prog, hint)) | 28 | (prog, hint)) |
| 24 | sys.exit(1) | 29 | sys.exit(1) |
| 25 | else: | 30 | else: |
| 26 | return None | 31 | return None |
| 27 | 32 | ||
| 33 | def get_config_option(option): | ||
| 34 | '''Search for @option in installed kernel config (if present). | ||
| 35 | Raise an IOError if the kernel config isn't found in /boot/.''' | ||
| 36 | uname = subprocess.check_output(["uname", "-r"])[-1] | ||
| 37 | fname = "/boot/config-%s" % uname | ||
| 38 | |||
| 39 | if os.path.exists(fname): | ||
| 40 | config_regex = "^CONFIG_{}=(?P<val>.*)$".format(option) | ||
| 41 | match = re.search(config_regex, open(fname, 'r').read()) | ||
| 42 | if not match: | ||
| 43 | return None | ||
| 44 | else: | ||
| 45 | return match.group("val") | ||
| 46 | |||
| 47 | else: | ||
| 48 | raise IOError("No config file exists!") | ||
| 49 | |||
| 28 | def recordtype(typename, field_names, default=0): | 50 | def recordtype(typename, field_names, default=0): |
| 29 | ''' Mutable namedtuple. Recipe from George Sakkis of MIT.''' | 51 | ''' Mutable namedtuple. Recipe from George Sakkis of MIT.''' |
| 30 | field_names = tuple(map(str, field_names)) | 52 | field_names = tuple(map(str, field_names)) |
diff --git a/config/config.py b/config/config.py index 3282705..d463999 100644 --- a/config/config.py +++ b/config/config.py | |||
| @@ -17,18 +17,20 @@ BINS = {'rtspin' : get_executable('rtspin', 'liblitmus'), | |||
| 17 | FILES = {'ft_data' : 'ft.bin', | 17 | FILES = {'ft_data' : 'ft.bin', |
| 18 | 'linux_data' : 'trace.dat', | 18 | 'linux_data' : 'trace.dat', |
| 19 | 'sched_data' : 'st-{}.bin', | 19 | 'sched_data' : 'st-{}.bin', |
| 20 | 'log_data' : 'trace.slog',} | 20 | 'log_data' : 'trace.slog'} |
| 21 | 21 | ||
| 22 | '''Default parameter names in params.py.''' | 22 | '''Default parameter names in params.py.''' |
| 23 | PARAMS = {'sched' : 'scheduler', | 23 | # TODO: add check for config options |
| 24 | 'dur' : 'duration', | 24 | PARAMS = {'sched' : 'scheduler', # Scheduler used by run_exps |
| 25 | 'kernel' : 'uname', | 25 | 'dur' : 'duration', # Duration of tests in run_exps |
| 26 | 'cycles' : 'cpu-frequency'} | 26 | 'kernel' : 'uname', # Regex of required OS name in run_exps |
| 27 | 'cycles' : 'cpu-frequency', # Frequency run_exps was run with | ||
| 28 | 'tasks' : 'tasks' # Number of tasks | ||
| 29 | } | ||
| 27 | 30 | ||
| 28 | '''Default values for program parameters.''' | 31 | '''Default values for program options.''' |
| 29 | DEFAULTS = {'params_file' : 'params.py', | 32 | DEFAULTS = {'params_file' : 'params.py', |
| 30 | 'sched_file' : 'sched.py', | 33 | 'sched_file' : 'sched.py', |
| 31 | 'exps_file' : 'exps.py', | ||
| 32 | 'duration' : 10, | 34 | 'duration' : 10, |
| 33 | 'spin' : 'rtspin', | 35 | 'spin' : 'rtspin', |
| 34 | 'cycles' : 2000} | 36 | 'cycles' : 2000} |
diff --git a/gen/__init__.py b/gen/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/gen/__init__.py | |||
diff --git a/gen/dp.py b/gen/dp.py new file mode 100644 index 0000000..0ac8cce --- /dev/null +++ b/gen/dp.py | |||
| @@ -0,0 +1,33 @@ | |||
| 1 | from __future__ import division | ||
| 2 | |||
| 3 | class DesignPointGenerator(object): | ||
| 4 | '''Iterates over all combinations of values specified in options. | ||
| 5 | Shamelessly stolen (and simplified) from bcw.''' | ||
| 6 | def __init__(self, options): | ||
| 7 | self.point_idx = 0 # Current point | ||
| 8 | self.options = options | ||
| 9 | self.total = 1 | ||
| 10 | for x in options.itervalues(): | ||
| 11 | self.total *= len(x) | ||
| 12 | |||
| 13 | def __iter__(self): | ||
| 14 | return self | ||
| 15 | |||
| 16 | def next(self): | ||
| 17 | while True: | ||
| 18 | if self.point_idx == self.total: | ||
| 19 | raise StopIteration | ||
| 20 | else: | ||
| 21 | point = {} | ||
| 22 | |||
| 23 | divisor = 1 | ||
| 24 | for key in sorted(self.options.keys()): | ||
| 25 | size = len(self.options[key]) | ||
| 26 | |||
| 27 | option_idx = int(self.point_idx / divisor) % size | ||
| 28 | point[key] = self.options[key][option_idx] | ||
| 29 | |||
| 30 | divisor *= size | ||
| 31 | self.point_idx += 1 | ||
| 32 | |||
| 33 | return point | ||
diff --git a/gen/generators.py b/gen/generators.py new file mode 100644 index 0000000..2fc77a7 --- /dev/null +++ b/gen/generators.py | |||
| @@ -0,0 +1,257 @@ | |||
| 1 | from Cheetah.Template import Template | ||
| 2 | from collections import namedtuple | ||
| 3 | from common import get_config_option | ||
| 4 | from config.config import DEFAULTS | ||
| 5 | from gen.dp import DesignPointGenerator | ||
| 6 | from parse.tuple_table import ColMap | ||
| 7 | |||
| 8 | import gen.rv as rv | ||
| 9 | import os | ||
| 10 | import random | ||
| 11 | import run.litmus_util as lu | ||
| 12 | import schedcat.generator.tasks as tasks | ||
| 13 | import shutil as sh | ||
| 14 | |||
| 15 | NAMED_PERIODS = { | ||
| 16 | 'harmonic' : rv.uniform_choice([25, 50, 100, 200]), | ||
| 17 | 'uni-short' : rv.uniform_int( 3, 33), | ||
| 18 | 'uni-moderate' : rv.uniform_int(10, 100), | ||
| 19 | 'uni-long' : rv.uniform_int(50, 250), | ||
| 20 | } | ||
| 21 | |||
| 22 | NAMED_UTILIZATIONS = { | ||
| 23 | 'uni-very-light': rv.uniform(0.0001, 0.001), | ||
| 24 | 'uni-light' : rv.uniform(0.001, 0.1), | ||
| 25 | 'uni-medium' : rv.uniform( 0.1, 0.4), | ||
| 26 | 'uni-heavy' : rv.uniform( 0.5, 0.9), | ||
| 27 | |||
| 28 | 'exp-light' : rv.exponential(0, 1, 0.10), | ||
| 29 | 'exp-medium' : rv.exponential(0, 1, 0.25), | ||
| 30 | 'exp-heavy' : rv.exponential(0, 1, 0.50), | ||
| 31 | |||
| 32 | 'bimo-light' : rv.multimodal([(rv.uniform(0.001, 0.5), 8), | ||
| 33 | (rv.uniform( 0.5, 0.9), 1)]), | ||
| 34 | 'bimo-medium' : rv.multimodal([(rv.uniform(0.001, 0.5), 6), | ||
| 35 | (rv.uniform( 0.5, 0.9), 3)]), | ||
| 36 | 'bimo-heavy' : rv.multimodal([(rv.uniform(0.001, 0.5), 4), | ||
| 37 | (rv.uniform( 0.5, 0.9), 5)]), | ||
| 38 | } | ||
| 39 | |||
| 40 | # Cheetah templates for schedule files | ||
| 41 | TP_CLUSTER = "plugins/C-EDF/cluster{$level}" | ||
| 42 | TP_RM = """#if $release_master | ||
| 43 | release_master{1} | ||
| 44 | #end if""" | ||
| 45 | TP_TBASE = """#for $t in $task_set | ||
| 46 | {}$t.cost $t.period | ||
| 47 | #end for""" | ||
| 48 | TP_PART_TASK = TP_TBASE.format("-p $t.cpu ") | ||
| 49 | TP_GLOB_TASK = TP_TBASE.format("") | ||
| 50 | |||
| 51 | GenOption = namedtuple('GenOption', ['name', 'types', 'default', 'help']) | ||
| 52 | |||
| 53 | class BaseGenerator(object): | ||
| 54 | '''Creates sporadic task sets with the most common Litmus options.''' | ||
| 55 | def __init__(self, name, templates, options, params): | ||
| 56 | self.options = self.__make_options() + options | ||
| 57 | |||
| 58 | self.__setup_params(params) | ||
| 59 | |||
| 60 | self.params = params | ||
| 61 | self.template = "\n".join([TP_RM] + templates) | ||
| 62 | self.name = name | ||
| 63 | |||
| 64 | def __make_options(self): | ||
| 65 | '''Return generic Litmus options.''' | ||
| 66 | |||
| 67 | # Guess defaults using the properties of this computer | ||
| 68 | cpus = lu.num_cpus() | ||
| 69 | try: | ||
| 70 | config = get_config_option("RELEASE_MASTER") and True | ||
| 71 | except: | ||
| 72 | config = False | ||
| 73 | release_master = list(set([False, config])) | ||
| 74 | |||
| 75 | list_types = [str, float, type([])] | ||
| 76 | |||
| 77 | return [GenOption('cpus', int, [cpus], | ||
| 78 | 'Number of processors on target system.'), | ||
| 79 | GenOption('num_tasks', int, range(cpus, 5*cpus, cpus), | ||
| 80 | 'Number of tasks per experiment.'), | ||
| 81 | GenOption('utils', list_types + NAMED_UTILIZATIONS.keys(), | ||
| 82 | ['uni-medium'],'Task utilization distributions.'), | ||
| 83 | GenOption('periods', list_types + NAMED_PERIODS.keys(), | ||
| 84 | ['harmonic'], 'Task period distributions.'), | ||
| 85 | GenOption('release_master', [True,False], release_master, | ||
| 86 | 'Redirect release interrupts to a single CPU.'), | ||
| 87 | GenOption('duration', float, [30], 'Experiment duration.')] | ||
| 88 | |||
| 89 | def __create_dist(self, name, value, named_dists): | ||
| 90 | '''Attempt to create a distribution representing the data in @value. | ||
| 91 | If @value is a string, use it as a key for @named_dists.''' | ||
| 92 | name = "%s distribution" % name | ||
| 93 | # A list of values | ||
| 94 | if type(value) == type([]): | ||
| 95 | map(lambda x : self.__check_value(name, x, [float, int]), value) | ||
| 96 | return rv.uniform_choice(value) | ||
| 97 | elif type(value) in [float, int]: | ||
| 98 | return lambda : value | ||
| 99 | elif value in named_dists: | ||
| 100 | return named_dists[value] | ||
| 101 | else: | ||
| 102 | raise ValueError("Invalid %s value: %s" % (name, value)) | ||
| 103 | |||
| 104 | def __create_exp(self, exp_params, out_dir): | ||
| 105 | '''Create a single experiment with @exp_params in @out_dir.''' | ||
| 106 | pdist = self.__create_dist('period', | ||
| 107 | exp_params['periods'], | ||
| 108 | NAMED_PERIODS) | ||
| 109 | udist = self.__create_dist('utilization', | ||
| 110 | exp_params['utils'], | ||
| 111 | NAMED_UTILIZATIONS) | ||
| 112 | tg = tasks.TaskGenerator(period=pdist, util=udist) | ||
| 113 | |||
| 114 | ts = [] | ||
| 115 | tries = 0 | ||
| 116 | while len(ts) != exp_params['num_tasks'] and tries < 5: | ||
| 117 | ts = tg.make_task_set(max_tasks = exp_params['num_tasks']) | ||
| 118 | tries += 1 | ||
| 119 | if len(ts) != exp_params['num_tasks']: | ||
| 120 | print("Failed to create task set with parameters: %s" % exp_params) | ||
| 121 | |||
| 122 | self._customize(ts, exp_params) | ||
| 123 | |||
| 124 | sched_file = out_dir + "/" + DEFAULTS['sched_file'] | ||
| 125 | with open(sched_file, 'wa') as f: | ||
| 126 | exp_params['task_set'] = ts | ||
| 127 | f.write(str(Template(self.template, searchList=[exp_params]))) | ||
| 128 | |||
| 129 | del exp_params['task_set'] | ||
| 130 | exp_params_file = out_dir + "/" + DEFAULTS['params_file'] | ||
| 131 | with open(exp_params_file, 'wa') as f: | ||
| 132 | exp_params['scheduler'] = 'CEDF' | ||
| 133 | f.write(str(exp_params)) | ||
| 134 | |||
| 135 | def __setup_params(self, params): | ||
| 136 | '''Set default parameter values and check that values are valid.''' | ||
| 137 | for option in self.options: | ||
| 138 | if option.name not in params: | ||
| 139 | params[option.name] = option.default | ||
| 140 | params[option.name] = self._check_value(option.name, | ||
| 141 | option.types, | ||
| 142 | params[option.name]) | ||
| 143 | return params | ||
| 144 | |||
| 145 | |||
| 146 | def _check_value(self, name, types, val): | ||
| 147 | '''Raise an exception if the value of type of @val is not specified | ||
| 148 | in @types. Returns a copy of @val with strings converted to raw | ||
| 149 | Python types, if possible.''' | ||
| 150 | if types == float: | ||
| 151 | types = [float, int] | ||
| 152 | if type(types) != type([]): | ||
| 153 | types = [types] | ||
| 154 | if type(val) != type([]): | ||
| 155 | val = [val] | ||
| 156 | |||
| 157 | retval = [] | ||
| 158 | for v in val: | ||
| 159 | # Has to be a better way to find this | ||
| 160 | v = False if v in ['f', 'False', 'false', 'n', 'no'] else v | ||
| 161 | v = True if v in ['t', 'True', 'true', 'y', 'yes'] else v | ||
| 162 | |||
| 163 | if type(v) not in types and v not in types: | ||
| 164 | # Try and convert v to one of the specified types | ||
| 165 | parsed = None | ||
| 166 | for t in types: | ||
| 167 | try: | ||
| 168 | parsed = t(v) | ||
| 169 | break | ||
| 170 | except: | ||
| 171 | pass | ||
| 172 | |||
| 173 | if parsed: | ||
| 174 | retval += [parsed] | ||
| 175 | else: | ||
| 176 | raise TypeError("Invalid %s value: '%s'" % (name, v)) | ||
| 177 | else: | ||
| 178 | retval += [v] | ||
| 179 | return retval | ||
| 180 | |||
| 181 | def _customize(self, taskset, exp_params): | ||
| 182 | '''Configure a generated taskset with extra parameters.''' | ||
| 183 | pass | ||
| 184 | |||
| 185 | def create_exps(self, out_dir, force): | ||
| 186 | '''Create experiments for all possible combinations of params in | ||
| 187 | @out_dir. Overwrite existing files if @force is True.''' | ||
| 188 | col_map = ColMap() | ||
| 189 | |||
| 190 | # Track changing values so only relevant parameters are included | ||
| 191 | # in directory names | ||
| 192 | for dp in DesignPointGenerator(self.params): | ||
| 193 | for k, v in dp.iteritems(): | ||
| 194 | col_map.try_add(k, v) | ||
| 195 | |||
| 196 | for dp in DesignPointGenerator(self.params): | ||
| 197 | dir_leaf = "sched=%s_%s" % (self.name, col_map.get_encoding(dp)) | ||
| 198 | dir_path = "%s/%s" % (out_dir, dir_leaf.strip('_')) | ||
| 199 | |||
| 200 | if os.path.exists(dir_path): | ||
| 201 | if force: | ||
| 202 | sh.rmtree(dir_path) | ||
| 203 | else: | ||
| 204 | print("Skipping existing experiment: '%s'" % dir_path) | ||
| 205 | continue | ||
| 206 | |||
| 207 | os.mkdir(dir_path) | ||
| 208 | |||
| 209 | self.__create_exp(dp, dir_path) | ||
| 210 | |||
| 211 | def print_help(self): | ||
| 212 | s = str(Template("""Generator $name: | ||
| 213 | #for $o in $options | ||
| 214 | $o.name -- $o.help | ||
| 215 | \tDefault: $o.default | ||
| 216 | \tAllowed: $o.types | ||
| 217 | #end for""", searchList=vars(self))) | ||
| 218 | |||
| 219 | # Has to be an easier way to print this out... | ||
| 220 | for line in s.split("\n"): | ||
| 221 | res = [] | ||
| 222 | i = 0 | ||
| 223 | for word in line.split(", "): | ||
| 224 | i+= len(word) | ||
| 225 | res += [word] | ||
| 226 | if i > 80: | ||
| 227 | print ", ".join(res[:-1]) | ||
| 228 | res = ["\t\t "+res[-1]] | ||
| 229 | i = line.index("'") | ||
| 230 | print ", ".join(res) | ||
| 231 | |||
| 232 | class PartitionedGenerator(BaseGenerator): | ||
| 233 | def __init__(self, name, templates, options, params): | ||
| 234 | super(PartitionedGenerator, self).__init__(name, | ||
| 235 | templates + [TP_PART_TASK], options, params) | ||
| 236 | |||
| 237 | def _customize(self, taskset, exp_params): | ||
| 238 | start = 1 if exp_params['release_master'] else 0 | ||
| 239 | # Random partition for now: could do a smart partitioning | ||
| 240 | for t in taskset: | ||
| 241 | t.cpu = random.randint(start, exp_params['cpus'] - 1) | ||
| 242 | |||
| 243 | class PedfGenerator(PartitionedGenerator): | ||
| 244 | def __init__(self, params={}): | ||
| 245 | super(PedfGenerator, self).__init__("P-EDF", [], [], params) | ||
| 246 | |||
| 247 | class CedfGenerator(PartitionedGenerator): | ||
| 248 | LEVEL_OPTION = GenOption('level', ['L2', 'L3', 'All'], ['L2'], | ||
| 249 | 'Cache clustering level.',) | ||
| 250 | |||
| 251 | def __init__(self, params={}): | ||
| 252 | super(CedfGenerator, self).__init__("C-EDF", [TP_CLUSTER], | ||
| 253 | [CedfGenerator.LEVEL_OPTION], params) | ||
| 254 | |||
| 255 | class GedfGenerator(BaseGenerator): | ||
| 256 | def __init__(self, params={}): | ||
| 257 | super(GedfGenerator, self).__init__("G-EDF", [TP_GLOB_TASK], [], params) | ||
diff --git a/gen/rv.py b/gen/rv.py new file mode 100644 index 0000000..e6f4d0f --- /dev/null +++ b/gen/rv.py | |||
| @@ -0,0 +1,86 @@ | |||
| 1 | from __future__ import division | ||
| 2 | import random | ||
| 3 | |||
| 4 | def uniform_int(minval, maxval): | ||
| 5 | "Create a function that draws ints uniformly from {minval, ..., maxval}" | ||
| 6 | def _draw(): | ||
| 7 | return random.randint(minval, maxval) | ||
| 8 | return _draw | ||
| 9 | |||
| 10 | def uniform(minval, maxval): | ||
| 11 | "Create a function that draws floats uniformly from [minval, maxval]" | ||
| 12 | def _draw(): | ||
| 13 | return random.uniform(minval, maxval) | ||
| 14 | return _draw | ||
| 15 | |||
| 16 | def bernoulli(p): | ||
| 17 | "Create a function that flips a weight coin with probability p" | ||
| 18 | def _draw(): | ||
| 19 | return random.random() < p | ||
| 20 | return _draw | ||
| 21 | |||
| 22 | def uniform_choice(choices): | ||
| 23 | "Create a function that draws uniformly elements from choices" | ||
| 24 | selector = uniform_int(0, len(choices) - 1) | ||
| 25 | def _draw(): | ||
| 26 | return choices[selector()] | ||
| 27 | return _draw | ||
| 28 | |||
| 29 | def truncate(minval, maxval): | ||
| 30 | def _limit(fun): | ||
| 31 | def _f(*args, **kargs): | ||
| 32 | val = fun(*args, **kargs) | ||
| 33 | return min(maxval, max(minval, val)) | ||
| 34 | return _f | ||
| 35 | return _limit | ||
| 36 | |||
| 37 | def redraw(minval, maxval): | ||
| 38 | def _redraw(dist): | ||
| 39 | def _f(*args, **kargs): | ||
| 40 | in_range = False | ||
| 41 | while not in_range: | ||
| 42 | val = dist(*args, **kargs) | ||
| 43 | in_range = minval <= val <= maxval | ||
| 44 | return val | ||
| 45 | return _f | ||
| 46 | return _redraw | ||
| 47 | |||
| 48 | def exponential(minval, maxval, mean, limiter=redraw): | ||
| 49 | """Create a function that draws floats from an exponential | ||
| 50 | distribution with expected value 'mean'. If a drawn value is less | ||
| 51 | than minval or greater than maxval, then either another value is | ||
| 52 | drawn (if limiter=redraw) or the drawn value is set to minval or | ||
| 53 | maxval (if limiter=truncate).""" | ||
| 54 | def _draw(): | ||
| 55 | return random.expovariate(1.0 / mean) | ||
| 56 | return limiter(minval, maxval)(_draw) | ||
| 57 | |||
| 58 | def multimodal(weighted_distributions): | ||
| 59 | """Create a function that draws values from several distributions | ||
| 60 | with probability according to the given weights in a list of | ||
| 61 | (distribution, weight) pairs.""" | ||
| 62 | total_weight = sum([w for (d, w) in weighted_distributions]) | ||
| 63 | selector = uniform(0, total_weight) | ||
| 64 | def _draw(): | ||
| 65 | x = selector() | ||
| 66 | wsum = 0 | ||
| 67 | for (d, w) in weighted_distributions: | ||
| 68 | wsum += w | ||
| 69 | if wsum >= x: | ||
| 70 | return d() | ||
| 71 | assert False # should never drop off | ||
| 72 | return _draw | ||
| 73 | |||
| 74 | def uniform_slack(min_slack_ratio, max_slack_ratio): | ||
| 75 | """Choose deadlines uniformly such that the slack | ||
| 76 | is within [cost + min_slack_ratio * (period - cost), | ||
| 77 | cost + max_slack_ratio * (period - cost)]. | ||
| 78 | |||
| 79 | Setting max_slack_ratio = 1 implies constrained deadlines. | ||
| 80 | """ | ||
| 81 | def choose_deadline(cost, period): | ||
| 82 | slack = period - cost | ||
| 83 | earliest = slack * min_slack_ratio | ||
| 84 | latest = slack * max_slack_ratio | ||
| 85 | return cost + random.uniform(earliest, latest) | ||
| 86 | return choose_deadline | ||
diff --git a/gen_exps.py b/gen_exps.py index e69de29..e4e8187 100644..100755 --- a/gen_exps.py +++ b/gen_exps.py | |||
| @@ -0,0 +1,98 @@ | |||
| 1 | #!/usr/bin/env python | ||
| 2 | from __future__ import print_function | ||
| 3 | |||
| 4 | import os | ||
| 5 | import re | ||
| 6 | import shutil as sh | ||
| 7 | |||
| 8 | from gen.generators import GedfGenerator,PedfGenerator,CedfGenerator | ||
| 9 | from optparse import OptionParser | ||
| 10 | |||
| 11 | # There has to be a better way to do this... | ||
| 12 | GENERATORS = {'C-EDF':CedfGenerator, | ||
| 13 | 'P-EDF':PedfGenerator, | ||
| 14 | 'G-EDF':GedfGenerator} | ||
| 15 | |||
| 16 | def parse_args(): | ||
| 17 | parser = OptionParser("usage: %prog [options] [files...] " | ||
| 18 | "[generators...] [param=val[,val]...]") | ||
| 19 | |||
| 20 | parser.add_option('-o', '--out-dir', dest='out_dir', | ||
| 21 | help='directory for data output', | ||
| 22 | default=("%s/exps"%os.getcwd())) | ||
| 23 | parser.add_option('-f', '--force', action='store_true', default=False, | ||
| 24 | dest='force', help='overwrite existing data') | ||
| 25 | parser.add_option('-l', '--list-generators', dest='list_gens', | ||
| 26 | help='list allowed generators', action='store_true', | ||
| 27 | default=False) | ||
| 28 | parser.add_option('-d', '--describe-generators', metavar='generator[,..]', | ||
| 29 | dest='described', default=None, | ||
| 30 | help='describe parameters for generator(s)') | ||
| 31 | |||
| 32 | return parser.parse_args() | ||
| 33 | |||
| 34 | def load_file(fname): | ||
| 35 | with open(fname, 'r') as f: | ||
| 36 | data = f.read().strip() | ||
| 37 | try: | ||
| 38 | values = eval(data) | ||
| 39 | if 'generator' not in values: | ||
| 40 | raise ValueError() | ||
| 41 | generator = values['generator'] | ||
| 42 | del values['generator'] | ||
| 43 | return generator, values | ||
| 44 | except: | ||
| 45 | raise IOError("Invalid generation file: %s" % fname) | ||
| 46 | |||
| 47 | def main(): | ||
| 48 | opts, args = parse_args() | ||
| 49 | |||
| 50 | # Print generator information on the command line | ||
| 51 | if opts.list_gens: | ||
| 52 | print(", ".join(GENERATORS.keys())) | ||
| 53 | if opts.described != None: | ||
| 54 | for generator in opts.described.split(','): | ||
| 55 | if generator not in GENERATORS: | ||
| 56 | print("No generator '%s'" % generator) | ||
| 57 | else: | ||
| 58 | GENERATORS[generator]().print_help() | ||
| 59 | if opts.list_gens or opts.described: | ||
| 60 | return 0 | ||
| 61 | |||
| 62 | params = filter(lambda x : re.match("\w+=\w+", x), args) | ||
| 63 | |||
| 64 | # Ensure some generator is loaded | ||
| 65 | args = list(set(args) - set(params)) | ||
| 66 | #TODO: get every loaded plugin, try and use that generator | ||
| 67 | args = args or ['C-EDF', 'G-EDF', 'P-EDF'] | ||
| 68 | |||
| 69 | # Split into files to load, named generators | ||
| 70 | files = filter(os.path.exists, args) | ||
| 71 | gen_list = list(set(args) - set(files)) | ||
| 72 | |||
| 73 | # Parse all specified parameters to be applied to every experiment | ||
| 74 | global_params = dict(map(lambda x : tuple(x.split("=")), params)) | ||
| 75 | for k, v in global_params.iteritems(): | ||
| 76 | global_params[k] = v.split(',') | ||
| 77 | |||
| 78 | exp_sets = map(load_file, files) | ||
| 79 | exp_sets += map(lambda x: (x, {}), gen_list) | ||
| 80 | |||
| 81 | if opts.force and os.path.exists(opts.out_dir): | ||
| 82 | sh.rmtree(opts.out_dir) | ||
| 83 | if not os.path.exists(opts.out_dir): | ||
| 84 | os.mkdir(opts.out_dir) | ||
| 85 | |||
| 86 | for gen_name, gen_params in exp_sets: | ||
| 87 | if gen_name not in GENERATORS: | ||
| 88 | raise ValueError("Invalid generator name: %s" % gen_name) | ||
| 89 | |||
| 90 | print("Creating experiments using %s generator..." % gen_name) | ||
| 91 | |||
| 92 | params = dict(gen_params.items() + global_params.items()) | ||
| 93 | generator = GENERATORS[gen_name](params) | ||
| 94 | |||
| 95 | generator.create_exps(opts.out_dir, opts.force) | ||
| 96 | |||
| 97 | if __name__ == '__main__': | ||
| 98 | main() | ||
diff --git a/parse/ft.py b/parse/ft.py index a6596b7..5293b00 100644 --- a/parse/ft.py +++ b/parse/ft.py | |||
| @@ -47,12 +47,12 @@ def parse_overhead(result, overhead_bin, overhead, cycles, out_dir, err_file): | |||
| 47 | 47 | ||
| 48 | def sort_ft(ft_file, err_file, out_dir): | 48 | def sort_ft(ft_file, err_file, out_dir): |
| 49 | '''Create and return file with sorted overheads from @ft_file.''' | 49 | '''Create and return file with sorted overheads from @ft_file.''' |
| 50 | out_fname = "{}/{}".format(out_dir, FT_SORTED_NAME) | 50 | out_fname = "{}/{}".format("%s/%s" % (os.getcwd(), out_dir), FT_SORTED_NAME) |
| 51 | 51 | ||
| 52 | # Sort happens in-place | 52 | # Sort happens in-place |
| 53 | sh.copyfile(ft_file, out_fname) | 53 | sh.copyfile(ft_file, out_fname) |
| 54 | cmd = [conf.BINS['ftsort'], out_fname] | 54 | cmd = [conf.BINS['ftsort'], out_fname] |
| 55 | ret = subprocess.call(cmd, cwd=out_dir, stderr=err_file, stdout=err_file) | 55 | ret = subprocess.call(cmd, cwd="%s/%s" % (os.getcwd(), out_dir), stderr=err_file, stdout=err_file) |
| 56 | 56 | ||
| 57 | if ret: | 57 | if ret: |
| 58 | raise Exception("Sort failed with command: %s" % " ".join(cmd)) | 58 | raise Exception("Sort failed with command: %s" % " ".join(cmd)) |
diff --git a/parse/sched.py b/parse/sched.py index 512ac73..ba0df5e 100644 --- a/parse/sched.py +++ b/parse/sched.py | |||
| @@ -146,7 +146,7 @@ def extract_sched_data(result, data_dir, work_dir): | |||
| 146 | bin_files = conf.FILES['sched_data'].format(".*") | 146 | bin_files = conf.FILES['sched_data'].format(".*") |
| 147 | output_file = "%s/out-st" % work_dir | 147 | output_file = "%s/out-st" % work_dir |
| 148 | 148 | ||
| 149 | bins = [f for f in os.listdir(data_dir) if re.match(bin_files, f)] | 149 | bins = ["%s/%s" % (data_dir,f) for f in os.listdir(data_dir) if re.match(bin_files, f)] |
| 150 | if not len(bins): | 150 | if not len(bins): |
| 151 | return | 151 | return |
| 152 | 152 | ||
diff --git a/parse/tuple_table.py b/parse/tuple_table.py index e5dc39b..86006d2 100644 --- a/parse/tuple_table.py +++ b/parse/tuple_table.py | |||
| @@ -23,6 +23,15 @@ class ColMap(object): | |||
| 23 | key += (kv[col],) | 23 | key += (kv[col],) |
| 24 | return key | 24 | return key |
| 25 | 25 | ||
| 26 | def get_encoding(self, kv): | ||
| 27 | def escape(val): | ||
| 28 | return str(val).replace("_", "-").replace("=", "-") | ||
| 29 | vals = [] | ||
| 30 | for key in self.col_list: | ||
| 31 | k, v = escape(key), escape(kv[key]) | ||
| 32 | vals += ["%s=%s" % (k, v)] | ||
| 33 | return "_".join(vals) | ||
| 34 | |||
| 26 | def __contains__(self, col): | 35 | def __contains__(self, col): |
| 27 | return col in self.rev_map | 36 | return col in self.rev_map |
| 28 | 37 | ||
diff --git a/parse_exps.py b/parse_exps.py index c8cd8b1..f7e1342 100755 --- a/parse_exps.py +++ b/parse_exps.py | |||
| @@ -18,6 +18,8 @@ def parse_args(): | |||
| 18 | # TODO: convert data-dir to proper option, clean 'dest' options | 18 | # TODO: convert data-dir to proper option, clean 'dest' options |
| 19 | parser = OptionParser("usage: %prog [options] [data_dir]...") | 19 | parser = OptionParser("usage: %prog [options] [data_dir]...") |
| 20 | 20 | ||
| 21 | print("default to no params.py") | ||
| 22 | |||
| 21 | parser.add_option('-o', '--out', dest='out', | 23 | parser.add_option('-o', '--out', dest='out', |
| 22 | help='file or directory for data output', default='parse-data') | 24 | help='file or directory for data output', default='parse-data') |
| 23 | parser.add_option('-c', '--clean', action='store_true', default=False, | 25 | parser.add_option('-c', '--clean', action='store_true', default=False, |
| @@ -41,19 +43,14 @@ def get_exp_params(data_dir, col_map): | |||
| 41 | if not os.path.isfile: | 43 | if not os.path.isfile: |
| 42 | raise Exception("No param file '%s' exists!" % param_file) | 44 | raise Exception("No param file '%s' exists!" % param_file) |
| 43 | 45 | ||
| 44 | # Ignore 'magic' parameters used by these scripts | ||
| 45 | params = load_params(param_file) | 46 | params = load_params(param_file) |
| 46 | for ignored in conf.PARAMS.itervalues(): | ||
| 47 | # With the exception of cycles which is used by overhead parsing | ||
| 48 | if ignored in params and ignored != conf.PARAMS['cycles']: | ||
| 49 | params.pop(ignored) | ||
| 50 | 47 | ||
| 51 | # Store parameters in col_map, which will track which parameters change | 48 | # Store parameters in col_map, which will track which parameters change |
| 52 | # across experiments | 49 | # across experiments |
| 53 | for key, value in params.iteritems(): | 50 | for key, value in params.iteritems(): |
| 54 | col_map.try_add(key, value) | 51 | col_map.try_add(key, value) |
| 55 | 52 | ||
| 56 | # Cycles must be present | 53 | # Cycles must be present for feather-trace measurement parsing |
| 57 | if conf.PARAMS['cycles'] not in params: | 54 | if conf.PARAMS['cycles'] not in params: |
| 58 | params[conf.PARAMS['cycles']] = conf.DEFAULTS['cycles'] | 55 | params[conf.PARAMS['cycles']] = conf.DEFAULTS['cycles'] |
| 59 | 56 | ||
| @@ -72,10 +69,10 @@ def load_exps(exp_dirs, col_map, clean): | |||
| 72 | # Used to store error output and debugging info | 69 | # Used to store error output and debugging info |
| 73 | work_dir = data_dir + "/tmp" | 70 | work_dir = data_dir + "/tmp" |
| 74 | 71 | ||
| 72 | if os.path.exists(work_dir) and clean: | ||
| 73 | sh.rmtree(work_dir) | ||
| 75 | if not os.path.exists(work_dir): | 74 | if not os.path.exists(work_dir): |
| 76 | os.mkdir(work_dir) | 75 | os.mkdir(work_dir) |
| 77 | elif clean: | ||
| 78 | sh.rmtree(work_dir) | ||
| 79 | 76 | ||
| 80 | params = get_exp_params(data_dir, col_map) | 77 | params = get_exp_params(data_dir, col_map) |
| 81 | 78 | ||
diff --git a/run_exps.py b/run_exps.py index cc348ec..8fd9ed2 100755 --- a/run_exps.py +++ b/run_exps.py | |||
| @@ -117,6 +117,7 @@ def load_experiment(sched_file, scheduler, duration, param_file, out_dir): | |||
| 117 | # Cycles is saved here for accurate overhead calculations later | 117 | # Cycles is saved here for accurate overhead calculations later |
| 118 | out_params = dict(params.items() + | 118 | out_params = dict(params.items() + |
| 119 | [(conf.PARAMS['sched'], scheduler), | 119 | [(conf.PARAMS['sched'], scheduler), |
| 120 | (conf.PARAMS['tasks'], len(schedule['spin'])), | ||
| 120 | (conf.PARAMS['dur'], duration), | 121 | (conf.PARAMS['dur'], duration), |
| 121 | (conf.PARAMS['cycles'], lu.cpu_freq())]) | 122 | (conf.PARAMS['cycles'], lu.cpu_freq())]) |
| 122 | with open("%s/%s" % (out_dir, conf.DEFAULTS['params_file']), 'w') as f: | 123 | with open("%s/%s" % (out_dir, conf.DEFAULTS['params_file']), 'w') as f: |
