Allow polluter/splitting span the same task set

2 files changed, 96 insertions, 61 deletions

diff --git a/gen/edf_generators.py b/gen/edf_generators.py
index 7a30b4f..33f52be 100644
--- a/gen/edf_generators.py
+++ b/gen/edf_generators.py
@@ -84,6 +84,9 @@ class EdfPgmGenerator(gen.Generator):
                 gen.Generator._dist_option('depth_factor', ['uni-medium'],
                                            ecrts14.NAMED_HEIGHT_FACTORS,
                                            'Depth of graphs.'),
+                gen.Generator._dist_option('clustering', ['L1', 'L2', 'L3', 'ALL'],
+                                            {},
+                                           'Clustering configurations'),
                 gen.Generator._dist_option('partitions', ['no_cache', 'parallel', 'cache_aware'],
                                            PARTITION_METHOD,
                                            'Partition methods.'),
@@ -105,13 +108,10 @@ class EdfPgmGenerator(gen.Generator):
                 gen.Generator._dist_option('task_util', ['uni-light'],
                                            NAMED_UTILIZATIONS,
                                            'Task utilization.'),
-                gen.Generator._dist_option('polluters', False,
+                gen.Generator._dist_option('polluters', [False, True],
                                            {},
                                            'Polluters.'),
-#                gen.Generator._dist_option('release_master', False,
-#                                           {},
-#                                           'Release master.'),
-                gen.Generator._dist_option('job_splitting', True,
+                gen.Generator._dist_option('job_splitting', [False, True],
                                            {},
                                            'Job splitting.'),
                 gen.Generator._dist_option('ovh_type', 'max',
@@ -247,6 +247,8 @@ class CflSplitPgmGenerator(EdfPgmGenerator):
         else:
             assert False
 
+        exp_params['fan_in_cap'] = int(exp_params['fan_in_cap'])
+
         dp.nr_clusters = cpus / cluster_sz
         assert dp.nr_clusters * cluster_sz == cpus
 
@@ -261,6 +263,7 @@ class CflSplitPgmGenerator(EdfPgmGenerator):
         # compute split factor
         working_ts = ts
         partitions = get_partitions(working_ts, dp.nr_clusters, cluster_sz)
-        is_srt_sched = split.compute_splits_nolock(overheads, False, working_ts, partitions, bypass_split = not dp.job_splitting)
+        do_splits = dp.job_splitting == 'True'
+        is_srt_sched = split.compute_splits_nolock(overheads, False, working_ts, partitions, bypass_split = not do_splits)
 
         return True, working_ts
diff --git a/gen/generator.py b/gen/generator.py
index 89b893e..7b254b5 100644
--- a/gen/generator.py
+++ b/gen/generator.py
@@ -3,6 +3,7 @@ import os
 import sys
 import copy
 import math
+import itertools
 import pprint
 import schedcat.generator.tasks as tasks
 import shutil as sh
@@ -361,17 +362,44 @@ class Generator(object):
         exp.sched = ['edf']
         exp.update(self.params)
         
+        # extract the parameters we want to test the same task set under
+        polluter_method = exp['polluters']
+        split_method = exp['job_splitting']
+        del exp['polluters']
+        del exp['job_splitting']
+
         # Track changing values so only relevant parameters are included
         # in directory names
         for dp in PgmDesignPointGenerator(exp):
             for k, v in dp.iteritems():
                 builder.try_add(k, v)
         col_map = builder.build()
-        
+
+        # extract the parameters we want to test the same task set under
         partition_method = exp['partitions']
+        cluster_method = exp['clustering']
         del exp['partitions']
+        del exp['clustering']
+
+        shared_params = []
+        for part, clust, pol, splt in list(itertools.product(partition_method, cluster_method, polluter_method, split_method)):
+            if clust == 'ALL' and part != 'no_cache':
+                # skip over partition methods when there is no clustering/partitioning
+                continue
+            p = storage()
+            p.partitioning = part
+            p.clustering = clust
+            p.polluting = pol
+            p.splitting = splt
+            shared_params.append(p)
 
         for _dp in PgmDesignPointGenerator(exp):
+
+            # TODO: Find out why fan_in_cap is set to a string. >:(
+            # Force it to be int.
+            for i,c in enumerate(_dp.fan_in_cap):
+                _dp.fan_in_cap = int(c)
+
             for trial in xrange(trials):
                 dp = copy.deepcopy(_dp)
                 dp.num_graphs = NAMED_NUM_GRAPHS[dp.num_graphs]
@@ -395,65 +423,69 @@ class Generator(object):
                     ts, graphs, subts = self._create_tasks(dp)
                     dp.tasks = len(ts)
 
-                    # TODO: Reuse the same task set and partition with polluter overheads
-                    # and with and without job splitting.
-                    # That is, same task set for:
-                    #   {poluters, no polluters} x {job splitting, no job splitting} x
-                    #   {cluster levels} x {partition methods}
-
-                    levels = ['L1', 'L2', 'L3', 'ALL']
                     try:
-                        for lvl in levels:
-                            dp['level'] = lvl
-                            _dp['level'] = lvl
-                            for pm in partition_method:
-                                dp.partitions = pm
-                                # Create directory name from relevant parameters
-                                dir_leaf  = "sched=%s_cluster=%s_%s" % (self.scheduler, lvl, col_map.encode(dp))
-                                dir_leaf  = dir_leaf.strip('_') # If there are none
-                                dir_leaf += ("_trial=%s" % trial) if trials > 1 else ""
-
-                                dir_path  = "%s/%s" % (out_dir, dir_leaf.strip('_'))
-
-                                print("Generating for %s" % dir_path)
-
-                                if os.path.exists(dir_path):
-                                    if force:
-                                        sh.rmtree(dir_path)
-                                    else:
-                                        print("Skipping existing experiment: '%s'" % dir_path)
-                                        continue
-
-                                os.mkdir(dir_path)
-                                created_dirs.append(dir_path)
-
-                                if trials > 1:
-                                    dp[PARAMS['trial']] = trial
-                                    _dp[PARAMS['trial']] = trial
-                                self.out_dir = dir_path
-
-                                _dp.system = topology.Topology(machines[dp.host])
-                                _dp.partitions = pm
-
-                                # Write a sched.py and param.py for each partition method
-                                ret = self._create_exp(_dp, ts, graphs, subts)
-                                if not ret:
-                                    print("Bin-packing fails for " + dir_leaf)
-                                    last_failed = dir_leaf
-                                    raise Exception("Failed to partition.")
-                                del(self.out_dir)
-                                if PARAMS['trial'] in dp:
-                                    del dp[PARAMS['trial']]
-                                    del _dp[PARAMS['trial']]
-                                # just generate one experiment for global
-                                if dp['level'] == 'ALL':
-                                    break
+                        for shp in shared_params:
+                            dp['level'] = shp.clustering
+                            _dp['level'] = shp.clustering
+
+                            # load in the shared parameters
+                            dp.partitions = shp.partitioning
+                            dp.cluster = shp.clustering
+                            dp.polluters = shp.polluting
+                            dp.job_splitting = shp.splitting
+
+                            # Create directory name from relevant parameters
+                            dir_parts = []
+                            dir_parts.append("sched=%s" % self.scheduler)
+                            dir_parts.append("cluster=%s" % shp.clustering)
+                            dir_parts.append("polluterovh=%s" % shp.polluting)
+                            dir_parts.append("splitting=%s" % shp.splitting)
+                            others = col_map.encode(dp)
+                            if others != "":
+                                dir_parts.append(others)
+                            if trials > 1:
+                                dir_parts.append("trial=%d" % trial)
+                            dir_leaf = "_".join(dir_parts)
+                            dir_path  = "%s/%s" % (out_dir, dir_leaf)
+
+                            print("Generating %s" % dir_leaf)
+
+                            if os.path.exists(dir_path):
+                                if force:
+                                    sh.rmtree(dir_path)
+                                else:
+                                    print("Skipping existing experiment: '%s'" % dir_path)
+                                    continue
+
+                            os.mkdir(dir_path)
+                            created_dirs.append(dir_path)
+
+                            if trials > 1:
+                                dp[PARAMS['trial']] = trial
+                                _dp[PARAMS['trial']] = trial
+                            self.out_dir = dir_path
+
+                            _dp.system = topology.Topology(machines[dp.host])
+                            _dp.partitions = dp.partitions
+                            _dp.polluters = dp.polluters
+                            _dp.job_splitting = dp.job_splitting
+
+                            # Write a sched.py and param.py for each partition method
+                            ret = self._create_exp(_dp, ts, graphs, subts)
+                            if not ret:
+                                print("Bin-packing fails for " + dir_leaf)
+                                last_failed = dir_leaf
+                                raise Exception("Failed to partition.")
+                            del(self.out_dir)
+                            if PARAMS['trial'] in dp:
+                                del dp[PARAMS['trial']]
+                                del _dp[PARAMS['trial']]
                         success = True
                     except Exception, e:
                         for d in created_dirs:
                             sh.rmtree(d)
-                if not success:
-                    print("Failed to generate experiments. Last failed: %s" % last_failed)
+                    if not success:
+                        print("Failed to generate experiment (%s). Try count = %d" % (last_failed, tries))
 
 
     def print_help(self):