Clean up PGM task set generation parameters.

Work originally by Namhoon.

3 files changed, 123 insertions, 85 deletions

diff --git a/gen/edf_generators.py b/gen/edf_generators.py
index 58cbb0d..d9e8b31 100644
--- a/gen/edf_generators.py
+++ b/gen/edf_generators.py
@@ -2,12 +2,15 @@ import generator as gen
 import random
 import ecrts14.partition as partition
 import schedcat.sched.split_heuristic as split
+import ecrts14.ecrts14 as ecrts14
 
 from ecrts14.ecrts14 import create_pgm_task_set
 from ecrts14.ecrts14 import get_overheads
 from config.config import FILES,PARAMS
 from schedcat.overheads.model import Overheads, CacheDelay, ConsumerOverheads, ProducerOverheads
 from ecrts14.tests import get_partitions
+from schedcat.generator.tasksets import NAMED_UTILIZATIONS
+from schedcat.mapping.binpack import DidNotFit
 
 TP_TBASE = """#for $t in $task_set
 {} $t.cost $t.period
@@ -72,20 +75,65 @@ class EdfPgmGenerator(gen.Generator):
 
     def __make_options(self):
         '''Return generic EDF options.'''
-        return [gen.Generator._dist_option('utils', 'uni-medium',
-                                           gen.NAMED_UTILIZATIONS,
-                                           'Task utilization distributions.'),
-                gen.Generator._dist_option('periods', 'harmonic',
-                                           gen.NAMED_PERIODS,
-                                           'Task period distributions.')]
+        return [gen.Generator._dist_option('period', ['uni-long'],
+                                           ecrts14.NAMED_PERIODS_US,
+                                           'Task period distributions.'),
+                gen.Generator._dist_option('num_graphs', 'uni-medium',
+                                           ecrts14.NAMED_NUM_GRAPHS,
+                                           'Number of graphs.'),
+                gen.Generator._dist_option('depth_factor', ['uni-medium'],
+                                           ecrts14.NAMED_HEIGHT_FACTORS,
+                                           'Depth of graphs.'),
+                gen.Generator._dist_option('partitions', ['no_cache', 'parallel', 'cache_aware'],
+                                           PARTITION_METHOD,
+                                           'Partition methods.'),
+                gen.Generator._dist_option('node_placement', ['uniform'],
+                                           ecrts14.NAMED_SHAPES,
+                                           'The node placement of a graph.'),
+                gen.Generator._dist_option('fan_out', ['uniform_3'],
+                                           ecrts14.NAMED_FAN,
+                                           'The number of out edges of a node.'),
+                gen.Generator._dist_option('fan_in_cap', [3],
+                                           {}, 
+                                           'The maximum number of in-edges of a node.'),
+                gen.Generator._dist_option('edge_distance', ['uniform_3'],
+                                           ecrts14.NAMED_EDGE_HOP,
+                                           'The number of hops for an edge.'),
+                gen.Generator._dist_option('wss', ['uni-medium'],
+                                           ecrts14.NAMED_EDGE_WSS,
+                                           'Working set size.'),
+                gen.Generator._dist_option('task_util', ['uni-light'],
+                                           NAMED_UTILIZATIONS,
+                                           'Task utilization.'),
+                gen.Generator._dist_option('polluters', False,
+                                           {},
+                                           'Polluters.'),
+                gen.Generator._dist_option('release_master', False,
+                                           {},
+                                           'Release master.'),
+                gen.Generator._dist_option('job_splitting', True,
+                                           {},
+                                           'Job splitting.'),
+                gen.Generator._dist_option('ovh_type', 'max',
+                                           {},
+                                           'Overhead type.'),
+                gen.Generator._dist_option('heur_aggressiveness', 0.75,
+                                           {},
+                                           'heur_aggressiveness.'),
+                gen.Generator._dist_option('sys_util', [20.0, 9.0, 8.0, 7.0],
+                                           {},
+                                           'System utilization.')]
 
     def _create_exp(self, dp, ts, graphs, subts):
         '''Create a single experiment with @exp_params in @out_dir.'''
 
-        ts = self._customize(ts, graphs, subts, dp)
-
-        self._write_pgm_schedule(dict(dp.items() + [('task_set', ts)] + [('graphs', graphs)] + [('sub_task_set', subts)]))
-        self._write_params(dict(dp.items() + [('num_tasks', len(ts))]))
+        ret, ts = self._customize(ts, graphs, subts, dp)
+        
+        if ret:
+            self._write_pgm_schedule(dict(dp.items() + [('task_set', ts)] + [('graphs', graphs)] + [('sub_task_set', subts)]))
+            self._write_params(dict(dp.items() + [('num_tasks', len(ts))]))
+        
+        return ret
 
     def _create_tasks(self, dp):
         '''Create a task set.'''
@@ -211,11 +259,14 @@ class CflSplitPgmGenerator(EdfPgmGenerator):
         overheads = get_overheads(dp, dp.system)
         # do the partition here
         ts = partition.clear_partitioning(ts)
-        ts = PARTITION_METHOD[exp_params['partitions']](ts, graphs, subts, cluster_sz, dp.nr_clusters, dp.system, dp.heur_aggressiveness, overheads)
+        try:
+            ts = PARTITION_METHOD[exp_params['partitions']](ts, graphs, subts, cluster_sz, dp.nr_clusters, dp.system, dp.heur_aggressiveness, overheads)
+        except DidNotFit:
+            return False, ts
 
         # compute split factor
         working_ts = ts
         partitions = get_partitions(working_ts, dp.nr_clusters, cluster_sz)
         is_srt_sched = split.compute_splits_nolock(overheads, False, working_ts, partitions, bypass_split = not dp.job_splitting)
 
-        return working_ts
+        return True, working_ts
diff --git a/gen/generator.py b/gen/generator.py
index 7456021..2d0fbef 100644
--- a/gen/generator.py
+++ b/gen/generator.py
@@ -15,6 +15,7 @@ from parse.col_map import ColMapBuilder
 from numpy import arange
 from schedcat.util.storage import storage
 from ecrts14.machines import machines
+from ecrts14.ecrts14 import NAMED_NUM_GRAPHS, NAMED_SHAPES, NAMED_HEIGHT_FACTORS, NAMED_FAN, NAMED_EDGE_HOP, NAMED_EDGE_WSS
 
 NAMED_PERIODS = {
     'harmonic'            : rv.uniform_choice([25, 50, 100, 200]),
@@ -81,9 +82,7 @@ class Generator(object):
         release_master = list(set([False, bool(rm_config)]))
 
 
-        return [GenOption('partitions', str, ['no_cache', 'parallel', 'cache_aware', 'cache_aware_edges', 'cache_aware_bfs', 'cache_aware_dfs'],
-                          'Partition methods.'),
-                GenOption('tasks', int, [0],
+        return [GenOption('tasks', int, [0],
                           'Number of tasks'),
                 GenOption('cpus', int, [cpus],
                           'Number of processors on target system.'),
@@ -154,7 +153,6 @@ class Generator(object):
             split_arg_t = []
 
             for n in g.nodes:
-                assert n.graph.id == g.id
                 cluster_arg_t.append('node_' + str(n.id) + ':' + str(n.task.partition))
                 exec_arg_t.append('node_' + str(n.id) + ':' + str(n.task.cost))
                 split_arg_t.append('node_' + str(n.id) + ':' + str(n.task.split))
@@ -192,7 +190,7 @@ class Generator(object):
         pgm_args = []
         for i in range(len(pgm_params['graphs'])):
             pgm_args_t = '';
-            pgm_args_t += '--wait --cluster ' + cluster_arg[i] + ' --clusterSize ' + clustersz_arg[i] + ' --name graph_' + str(pgm_params['graphs'][i].id)
+            pgm_args_t += '--wait --cluster ' + cluster_arg[i] + ' --clusterSize ' + clustersz_arg[i] + ' --name graph_' + str(i)
             pgm_args_t += ' --graph ' + graph_desc_arg[i] + ' --rates ' + rates_arg[i] + ' --execution ' + exec_arg[i]
             pgm_args_t += ' --split ' + split_arg[i]
             if len(wss_arg[i]) != 0:
@@ -334,79 +332,78 @@ class Generator(object):
         out_dir = opts.out_dir
         force = opts.force
         trials = opts.trials
-
-        # hard coded here
+        
+        # Hardcoded design points
         exp = storage()
         exp.host = ['ludwig']
         cpus = 24.0
         exp.processors = [cpus]
-        exp.task_util = ['uni-medium']
-        exp.period = ['uni-long']
-        exp.sched = ['edf']
-        exp.sys_util = [ 20.0 ] #arange(1, cpus+0.1, 0.1)
-
         exp.wcycle = [ 0 ]
         exp.walk = ['seq']
         exp.huge_pages = [False]
         exp.uncached = [False]
-        exp.polluters = [False]
-        exp.ovh_type = ['max']
-        exp.release_master = [False]
-        exp.heur_aggressiveness = [0.75]
-        exp.job_splitting = [True]
+        exp.sched = ['edf']
         exp.update(self.params)
-
+        
         # Track changing values so only relevant parameters are included
         # in directory names
         for dp in PgmDesignPointGenerator(exp):
             for k, v in dp.iteritems():
                 builder.try_add(k, v)
         col_map = builder.build()
+        
+        partition_method = exp['partitions']
+        del exp['partitions']
 
-        for trial in xrange(trials):
-            dp.num_graphs = graph.uniform(opts.num_graphs, opts.num_graphs)
-            dp.depth_factor = [1.0/3.0, 2.0/3.0]
-            dp.node_placement = graph.uniform(opts.node_placement, opts.node_placement)
-            dp.fan_out = graph.geometric(1, opts.fan_out)
-            dp.fan_in_cap = opts.fan_in_cap
-            dp.edge_distance = graph.geometric(1, opts.edge_distance)
-            dp.nr_source = graph.uniform(opts.nr_source, opts.nr_source)
-            dp.nr_sink = graph.uniform(opts.nr_sink, opts.nr_sink)
-            dp.wss = tasks.multimodal([(tasks.uniform_int(1,2), 6), (tasks.uniform_int(2, 8), 3)])
-
-            # Generate a task set
-            ts, graphs, subts = self._create_tasks(dp)
-            dp.tasks = len(ts)
-
-            for dp in PgmDesignPointGenerator(exp):
-                # Create directory name from relevant parameters
-                dir_leaf  = "sched=%s_%s" % (self.scheduler, col_map.encode(dp))
-                dir_leaf  = dir_leaf.strip('_') # If there are none
-                dir_leaf += ("_trial=%s" % trial) if trials > 1 else ""
-
-                dir_path  = "%s/%s" % (out_dir, dir_leaf.strip('_'))
-
-                if os.path.exists(dir_path):
-                    if force:
-                        sh.rmtree(dir_path)
-                    else:
-                        print("Skipping existing experiment: '%s'" % dir_path)
-                        continue
-
-                os.mkdir(dir_path)
-
-                if trials > 1:
-                    dp[PARAMS['trial']] = trial
-                self.out_dir = dir_path
-
-                dp.system = topology.Topology(machines[dp.host])
-
-                # Write a sched.py and param.py for each partition method
-                self._create_exp(dp, ts, graphs, subts)
-
-                del(self.out_dir)
-                if PARAMS['trial'] in dp:
-                    del dp[PARAMS['trial']]
+        for dp in PgmDesignPointGenerator(exp):
+            for trial in xrange(trials):
+                dp.num_graphs = NAMED_NUM_GRAPHS[dp.num_graphs]
+                dp.depth_factor = NAMED_HEIGHT_FACTORS[dp.depth_factor]
+                dp.node_placement = NAMED_SHAPES[dp.node_placement]
+                dp.fan_out = NAMED_FAN[dp.fan_out]
+                dp.edge_distance = NAMED_EDGE_HOP[dp.edge_distance]
+                dp.nr_source = graph.uniform(opts.nr_source, opts.nr_source)
+                dp.nr_sink = graph.uniform(opts.nr_sink, opts.nr_sink)
+                dp.wss = NAMED_EDGE_WSS[dp.wss]
+                
+                # Generate a task set
+                ts, graphs, subts = self._create_tasks(dp)
+                dp.tasks = len(ts)
+
+                #for dp in PgmDesignPointGenerator(exp):
+                for pm in partition_method:
+                    dp.partitions = pm
+                    # Create directory name from relevant parameters
+                    dir_leaf  = "sched=%s_%s" % (self.scheduler, col_map.encode(dp))
+                    dir_leaf  = dir_leaf.strip('_') # If there are none
+                    dir_leaf += ("_trial=%s" % trial) if trials > 1 else ""
+
+                    dir_path  = "%s/%s" % (out_dir, dir_leaf.strip('_'))
+
+                    if os.path.exists(dir_path):
+                        if force:
+                            sh.rmtree(dir_path)
+                        else:
+                            print("Skipping existing experiment: '%s'" % dir_path)
+                            continue
+
+                    os.mkdir(dir_path)
+
+                    if trials > 1:
+                        dp[PARAMS['trial']] = trial
+                    self.out_dir = dir_path
+
+                    dp.system = topology.Topology(machines[dp.host])
+
+                    # Write a sched.py and param.py for each partition method
+                    ret = self._create_exp(dp, ts, graphs, subts)
+                    if not ret:
+                        print("Bin-packing fails for " + dir_leaf)
+                        os.rmdir(dir_path)
+
+                    del(self.out_dir)
+                    if PARAMS['trial'] in dp:
+                        del dp[PARAMS['trial']]
 
     def print_help(self):
         display_options = [o for o in self.options if not o.hidden]
diff --git a/gen_pgm_exps.py b/gen_pgm_exps.py
index 62723ec..c6f6198 100755
--- a/gen_pgm_exps.py
+++ b/gen_pgm_exps.py
@@ -30,19 +30,9 @@ def parse_args():
     parser.add_option('-d', '--describe-generators', metavar='generator[,..]',
                       dest='described', default=None,
                       help='describe parameters for generator(s)')
-    parser.add_option('-m', '--num-graphs', default=24, type='int', dest='num_graphs',
-                                          help='number of graphs for a taskset')
-    parser.add_option('-p', '--node-placement', default=1, type='int', dest='node_placement',
-                      help='node placement of the graph')
-    parser.add_option('-t', '--fan-out', default=3, type='int', dest='fan_out',
-                      help='fan out of a node')
-    parser.add_option('-i', '--fan-in-cap', default=3, type='int', dest='fan_in_cap',
-                      help='fan in cap')
-    parser.add_option('-g', '--edge-distance', default=3, type='int', dest='edge_distance',
-                      help='edge distance')
     parser.add_option('-u', '--nr-source', default=1, type='int', dest='nr_source',
                       help='number of source nodes')
-    parser.add_option('-v', '--nr_sink', default=1, type='int', dest='nr_sink',
+    parser.add_option('-v', '--nr-sink', default=1, type='int', dest='nr_sink',
                       help='number of sink nodes')