6 files changed, 93 insertions, 49 deletions

diff --git a/common.py b/common.py
index cba31bf..a26ac94 100644
--- a/common.py
+++ b/common.py
@@ -50,6 +50,12 @@ def get_config_option(option):
     else:
         raise IOError("No config file exists!")
 
+def try_get_config_option(option, default):
+    try:
+        get_config_option(option)
+    except:
+        return default
+
 def recordtype(typename, field_names, default=0):
     ''' Mutable namedtuple. Recipe from George Sakkis of MIT.'''
     field_names = tuple(map(str, field_names))
@@ -127,10 +133,7 @@ def load_params(fname):
     with open(fname, 'r') as f:
         data = f.read()
     try:
-        parsed = eval(data)
-        # Convert to defaultdict
-        for k in parsed:
-            params[k] = str(parsed[k])
+        params = eval(data)
     except Exception as e:
         raise IOError("Invalid param file: %s\n%s" % (fname, e))
 
diff --git a/gen/edf_generators.py b/gen/edf_generators.py
index c7267f7..9bf6b8f 100644
--- a/gen/edf_generators.py
+++ b/gen/edf_generators.py
@@ -1,15 +1,20 @@
 import generator as gen
 import random
-import schedcat.generator.tasks as tasks
+
+TP_TBASE = """#for $t in $task_set
+{} $t.cost $t.period
+#end for"""
+TP_GLOB_TASK = TP_TBASE.format("")
+TP_PART_TASK = TP_TBASE.format("-p $t.cpu")
 
 class EdfGenerator(gen.Generator):
     '''Creates sporadic task sets with the most common Litmus options.'''
     def __init__(self, name, templates, options, params):
         super(EdfGenerator, self).__init__(name, templates,
-                                           self.__make_options(params) + options,
+                                           self.__make_options() + options,
                                            params)
 
-    def __make_options(self, params):
+    def __make_options(self):
         '''Return generic EDF options.'''
         return [gen.Generator._dist_option('utils', ['uni-medium'],
                                            gen.NAMED_UTILIZATIONS,
@@ -26,23 +31,12 @@ class EdfGenerator(gen.Generator):
         udist = self._create_dist('utilization',
                                   exp_params['utils'],
                                   gen.NAMED_UTILIZATIONS)
-        tg = tasks.TaskGenerator(period=pdist, util=udist)
 
-        ts = []
-        tries = 0
-        while len(ts) != exp_params['num_tasks'] and tries < 5:
-            ts = tg.make_task_set(max_tasks = exp_params['num_tasks'])
-            tries += 1
-        if len(ts) != exp_params['num_tasks']:
-            print("Failed to create task set with parameters: %s" % exp_params)
+        ts = self._create_taskset(exp_params, pdist, udist)
 
         self._customize(ts, exp_params)
 
-        exp_params['task_set'] = ts
-        self._write_schedule(exp_params)
-
-        del exp_params['task_set']
-        del exp_params['num_tasks']
+        self._write_schedule(dict(exp_params.items() + ('task_set', ts)))
         self._write_params(exp_params)
 
     def _customize(self, taskset, exp_params):
@@ -53,7 +47,7 @@ class EdfGenerator(gen.Generator):
 class PartitionedGenerator(EdfGenerator):
     def __init__(self, name, templates, options, params):
         super(PartitionedGenerator, self).__init__(name,
-            templates + [gen.TP_PART_TASK], options, params)
+            templates + [TP_PART_TASK], options, params)
 
     def _customize(self, taskset, exp_params):
         start = 1 if exp_params['release_master'] else 0
@@ -78,5 +72,5 @@ class CedfGenerator(PartitionedGenerator):
 
 class GedfGenerator(EdfGenerator):
     def __init__(self, params={}):
-        super(GedfGenerator, self).__init__("GSN-EDF", [gen.TP_GLOB_TASK],
+        super(GedfGenerator, self).__init__("GSN-EDF", [TP_GLOB_TASK],
                                             [], params)
diff --git a/gen/generator.py b/gen/generator.py
index 8c3048b..3a6524d 100644
--- a/gen/generator.py
+++ b/gen/generator.py
@@ -1,10 +1,11 @@
 import gen.rv as rv
 import os
+import pprint
+import schedcat.generator.tasks as tasks
 import shutil as sh
 
 from Cheetah.Template import Template
-from collections import namedtuple
-from common import get_config_option,num_cpus
+from common import get_config_option,num_cpus,recordtype
 from config.config import DEFAULTS,PARAMS
 from gen.dp import DesignPointGenerator
 from parse.col_map import ColMapBuilder
@@ -21,6 +22,7 @@ NAMED_UTILIZATIONS = {
     'uni-light'     : rv.uniform(0.001, 0.1),
     'uni-medium'    : rv.uniform(  0.1, 0.4),
     'uni-heavy'     : rv.uniform(  0.5, 0.9),
+    'uni-mixed'     : rv.uniform(0.001, .4),
 
     'exp-light'     : rv.exponential(0, 1, 0.10),
     'exp-medium'    : rv.exponential(0, 1, 0.25),
@@ -36,15 +38,12 @@ NAMED_UTILIZATIONS = {
 
 '''Components of Cheetah template for schedule file'''
 TP_RM = """#if $release_master
-release_master{1}
+release_master{0}
 #end if"""
-TP_TBASE = """#for $t in $task_set
-{} $t.cost $t.period
-#end for"""
-TP_GLOB_TASK = TP_TBASE.format("")
-TP_PART_TASK = TP_TBASE.format("-p $t.cpu")
 
-GenOption = namedtuple('GenOption', ['name', 'types', 'default', 'help'])
+GenOptionT = recordtype('GenOption', ['name', 'types', 'default', 'help', 'hidden'])
+def GenOption(name, types, default, help, hidden = False):
+    return GenOptionT(name, types, default, help, hidden)
 
 class Generator(object):
     '''Creates all combinations @options specified by @params.
@@ -92,36 +91,65 @@ class Generator(object):
     def _create_dist(self, name, value, named_dists):
         '''Attempt to create a distribution representing the data in @value.
         If @value is a string, use it as a key for @named_dists.'''
-        name = "%s distribution" % name
         # A list of values
         if type(value) == type([]):
             map(lambda x : self.__check_value(name, x, [float, int]), value)
             return rv.uniform_choice(value)
         elif type(value) in [float, int]:
             return lambda : value
-        elif value in named_dists:
+        elif named_dists and value in named_dists:
             return named_dists[value]
         else:
             raise ValueError("Invalid %s value: %s" % (name, value))
 
+    def _create_taskset(self, params, periods, utils, max_util = None):
+        tg = tasks.TaskGenerator(period=periods, util=utils)
+        ts = []
+        tries = 0
+        while len(ts) != params['num_tasks'] and tries < 100:
+            ts = tg.make_task_set(max_tasks = params['num_tasks'], max_util=max_util)
+            tries += 1
+        if len(ts) != params['num_tasks']:
+            print(("Only created task set of size %d < %d for params %s. " +
+                   "Switching to light utilization.") %
+                  (len(ts), params['num_tasks'], params))
+            print("Switching to light util. This usually means the " +
+                  "utilization distribution is too agressive.")
+            return self._create_taskset(params, periods, NAMED_UTILIZATIONS['uni-light'],
+                                        max_util)
+        return ts
+
     def _write_schedule(self, params):
         '''Write schedule file using current template for @params.'''
         sched_file = self.out_dir + "/" + DEFAULTS['sched_file']
         with open(sched_file, 'wa') as f:
             f.write(str(Template(self.template, searchList=[params])))
 
+
     def _write_params(self, params):
         '''Write out file with relevant parameters.'''
+        # Don't include this in the parameters. It will be automatically added
+        # in run_exps.py
+        if 'num_tasks' in params:
+            num_tasks = params.pop('num_tasks')
+        else:
+            num_tasks = 0
+
         exp_params_file = self.out_dir + "/" + DEFAULTS['params_file']
         with open(exp_params_file, 'wa') as f:
             params['scheduler'] = self.name
-            f.write(str(params))
+            pprint.pprint(params, f)
+
+        if num_tasks:
+            params['num_tasks'] = num_tasks
 
     def __setup_params(self, params):
         '''Set default parameter values and check that values are valid.'''
         for option in self.options:
             if option.name not in params:
                 params[option.name] = option.default
+            else:
+                option.hidden = True
             params[option.name] = self._check_value(option.name,
                                                     option.types,
                                                     params[option.name])
@@ -207,14 +235,16 @@ class Generator(object):
                 if PARAMS['trial'] in dp:
                     del dp[PARAMS['trial']]
 
+    HELP_INDENT = 17
 
     def print_help(self):
+        display_options = [o for o in self.options if not o.hidden]
         s = str(Template("""Generator $name:
         #for $o in $options
         $o.name -- $o.help
         \tDefault: $o.default
         \tAllowed: $o.types
-        #end for""", searchList=vars(self)))
+        #end for""", searchList={'name':self.name, 'options':display_options}))
 
         # Has to be an easier way to print this out...
         for line in s.split("\n"):
@@ -223,8 +253,8 @@ class Generator(object):
             for word in line.split(", "):
                 i += len(word)
                 res += [word]
-                if i > 80:
+                if i > 80 and len(word) < 80:
                     print(", ".join(res[:-1]))
-                    res = ["\t\t "+res[-1]]
-                    i = line.index("'")
+                    res = [" "*Generator.HELP_INDENT +res[-1]]
+                    i = Generator.HELP_INDENT + len(word)
             print(", ".join(res))
diff --git a/gen_exps.py b/gen_exps.py
index 20a6c6f..cb05fe7 100755
--- a/gen_exps.py
+++ b/gen_exps.py
@@ -6,12 +6,14 @@ import re
 import shutil as sh
 
 from gen.edf_generators import GedfGenerator,PedfGenerator,CedfGenerator
+from gen.mc_generators import McGenerator
 from optparse import OptionParser
 
 # There has to be a better way to do this...
 GENERATORS = {'C-EDF':CedfGenerator,
               'P-EDF':PedfGenerator,
-              'G-EDF':GedfGenerator}
+              'G-EDF':GedfGenerator,
+              'MC':McGenerator}
 
 def parse_args():
     parser = OptionParser("usage: %prog [options] [files...] "
diff --git a/run/proc_entry.py b/run/proc_entry.py
index 0b7f9ce..4ac2c51 100644
--- a/run/proc_entry.py
+++ b/run/proc_entry.py
@@ -8,5 +8,8 @@ class ProcEntry(object):
     def write_proc(self):
         if not os.path.exists(self.proc):
             raise Exception("Invalid proc entry %s" % self.proc)
-        with open(self.proc, 'w') as entry:
-            entry.write(self.data)
+        try:
+            with open(self.proc, 'w') as entry:
+                entry.write(self.data)
+        except:
+            print("Failed to write into %s value:\n%s" % (self.proc, self.data))
diff --git a/run_exps.py b/run_exps.py
index 6873877..4a2d8ab 100755
--- a/run_exps.py
+++ b/run_exps.py
@@ -27,15 +27,24 @@ class InvalidConfig(Exception):
         self.results = results
 
     def __str__(self):
-        rstr = "'%s' - wanted: '%s', found: %s"
-        result = [rstr % (r.actual, r.param, r.wanted) for r in self.results]
-        return "Invalid kernel configuration\n" + result.join("\n")
+        rstr = "'%s'%swanted: '%s', found: %s"
+        messages = []
+        for r in self.results:
+            # For pretty alignment
+            tabs = (3 - len(r.param)/8)
+            messages += [rstr % (r.param, '\t'*tabs, r.wanted, r.actual)]
+
+        return "Invalid kernel configuration " +\
+               "(ignore configuration with -i option).\n" + "\n".join(messages)
 
 def parse_args():
     parser = OptionParser("usage: %prog [options] [sched_file]... [exp_dir]...")
 
     parser.add_option('-s', '--scheduler', dest='scheduler',
                       help='scheduler for all experiments')
+    parser.add_option('-i', '--ignore-environment', dest='ignore',
+                      action='store_true', default=False,
+                      help='run experiments even in invalid environments ')
     parser.add_option('-d', '--duration', dest='duration', type='int',
                       help='duration (seconds) of tasks')
     parser.add_option('-o', '--out-dir', dest='out_dir',
@@ -87,7 +96,7 @@ def fix_paths(schedule, exp_dir, sched_file):
             if os.path.exists(abspath):
                 args = args.replace(arg, abspath)
                 break
-            elif re.match(r'.*\w+\.\w+', arg):
+            elif re.match(r'.*\w+\.[a-zA-Z]\w*', arg):
                 print("WARNING: non-existent file '%s' may be referenced:\n\t%s"
                       % (arg, sched_file))
 
@@ -108,9 +117,10 @@ def verify_environment(kernel, copts):
                 results += [ConfigResult(param, wanted, actual)]
 
         if results:
-            raise InvalidKernel(results)
+            raise InvalidConfig(results)
 
-def load_experiment(sched_file, scheduler, duration, param_file, out_dir):
+def load_experiment(sched_file, scheduler, duration,
+                    param_file, out_dir, ignore):
     if not os.path.isfile(sched_file):
         raise IOError("Cannot find schedule file: %s" % sched_file)
 
@@ -146,7 +156,8 @@ def load_experiment(sched_file, scheduler, duration, param_file, out_dir):
 
     fix_paths(schedule, os.path.split(sched_file)[0], sched_file)
 
-    verify_environment(kernel, copts)
+    if not ignore:
+        verify_environment(kernel, copts)
 
     run_exp(exp_name, schedule, scheduler, kernel, duration, work_dir, out_dir)
 
@@ -251,7 +262,8 @@ def main():
             path = "%s/%s" % (path, opts.sched_file)
 
         try:
-            load_experiment(path, scheduler, duration, param_file, out_dir)
+            load_experiment(path, scheduler, duration, param_file,
+                            out_dir, opts.ignore)
             succ += 1
         except ExperimentDone:
             done += 1