From 424917db79a1e8855c5e867bcc602476899fa28e Mon Sep 17 00:00:00 2001 From: Glenn Elliott Date: Sat, 18 Jan 2014 16:20:45 -0500 Subject: Allow polluter/splitting span the same task set --- gen/edf_generators.py | 15 +++--- gen/generator.py | 142 +++++++++++++++++++++++++++++++------------------- 2 files changed, 96 insertions(+), 61 deletions(-) diff --git a/gen/edf_generators.py b/gen/edf_generators.py index 7a30b4f..33f52be 100644 --- a/gen/edf_generators.py +++ b/gen/edf_generators.py @@ -84,6 +84,9 @@ class EdfPgmGenerator(gen.Generator): gen.Generator._dist_option('depth_factor', ['uni-medium'], ecrts14.NAMED_HEIGHT_FACTORS, 'Depth of graphs.'), + gen.Generator._dist_option('clustering', ['L1', 'L2', 'L3', 'ALL'], + {}, + 'Clustering configurations'), gen.Generator._dist_option('partitions', ['no_cache', 'parallel', 'cache_aware'], PARTITION_METHOD, 'Partition methods.'), @@ -105,13 +108,10 @@ class EdfPgmGenerator(gen.Generator): gen.Generator._dist_option('task_util', ['uni-light'], NAMED_UTILIZATIONS, 'Task utilization.'), - gen.Generator._dist_option('polluters', False, + gen.Generator._dist_option('polluters', [False, True], {}, 'Polluters.'), -# gen.Generator._dist_option('release_master', False, -# {}, -# 'Release master.'), - gen.Generator._dist_option('job_splitting', True, + gen.Generator._dist_option('job_splitting', [False, True], {}, 'Job splitting.'), gen.Generator._dist_option('ovh_type', 'max', @@ -247,6 +247,8 @@ class CflSplitPgmGenerator(EdfPgmGenerator): else: assert False + exp_params['fan_in_cap'] = int(exp_params['fan_in_cap']) + dp.nr_clusters = cpus / cluster_sz assert dp.nr_clusters * cluster_sz == cpus @@ -261,6 +263,7 @@ class CflSplitPgmGenerator(EdfPgmGenerator): # compute split factor working_ts = ts partitions = get_partitions(working_ts, dp.nr_clusters, cluster_sz) - is_srt_sched = split.compute_splits_nolock(overheads, False, working_ts, partitions, bypass_split = not dp.job_splitting) + do_splits = dp.job_splitting == 'True' + is_srt_sched = split.compute_splits_nolock(overheads, False, working_ts, partitions, bypass_split = not do_splits) return True, working_ts diff --git a/gen/generator.py b/gen/generator.py index 89b893e..7b254b5 100644 --- a/gen/generator.py +++ b/gen/generator.py @@ -3,6 +3,7 @@ import os import sys import copy import math +import itertools import pprint import schedcat.generator.tasks as tasks import shutil as sh @@ -361,17 +362,44 @@ class Generator(object): exp.sched = ['edf'] exp.update(self.params) + # extract the parameters we want to test the same task set under + polluter_method = exp['polluters'] + split_method = exp['job_splitting'] + del exp['polluters'] + del exp['job_splitting'] + # Track changing values so only relevant parameters are included # in directory names for dp in PgmDesignPointGenerator(exp): for k, v in dp.iteritems(): builder.try_add(k, v) col_map = builder.build() - + + # extract the parameters we want to test the same task set under partition_method = exp['partitions'] + cluster_method = exp['clustering'] del exp['partitions'] + del exp['clustering'] + + shared_params = [] + for part, clust, pol, splt in list(itertools.product(partition_method, cluster_method, polluter_method, split_method)): + if clust == 'ALL' and part != 'no_cache': + # skip over partition methods when there is no clustering/partitioning + continue + p = storage() + p.partitioning = part + p.clustering = clust + p.polluting = pol + p.splitting = splt + shared_params.append(p) for _dp in PgmDesignPointGenerator(exp): + + # TODO: Find out why fan_in_cap is set to a string. >:( + # Force it to be int. + for i,c in enumerate(_dp.fan_in_cap): + _dp.fan_in_cap = int(c) + for trial in xrange(trials): dp = copy.deepcopy(_dp) dp.num_graphs = NAMED_NUM_GRAPHS[dp.num_graphs] @@ -395,65 +423,69 @@ class Generator(object): ts, graphs, subts = self._create_tasks(dp) dp.tasks = len(ts) - # TODO: Reuse the same task set and partition with polluter overheads - # and with and without job splitting. - # That is, same task set for: - # {poluters, no polluters} x {job splitting, no job splitting} x - # {cluster levels} x {partition methods} - - levels = ['L1', 'L2', 'L3', 'ALL'] try: - for lvl in levels: - dp['level'] = lvl - _dp['level'] = lvl - for pm in partition_method: - dp.partitions = pm - # Create directory name from relevant parameters - dir_leaf = "sched=%s_cluster=%s_%s" % (self.scheduler, lvl, col_map.encode(dp)) - dir_leaf = dir_leaf.strip('_') # If there are none - dir_leaf += ("_trial=%s" % trial) if trials > 1 else "" - - dir_path = "%s/%s" % (out_dir, dir_leaf.strip('_')) - - print("Generating for %s" % dir_path) - - if os.path.exists(dir_path): - if force: - sh.rmtree(dir_path) - else: - print("Skipping existing experiment: '%s'" % dir_path) - continue - - os.mkdir(dir_path) - created_dirs.append(dir_path) - - if trials > 1: - dp[PARAMS['trial']] = trial - _dp[PARAMS['trial']] = trial - self.out_dir = dir_path - - _dp.system = topology.Topology(machines[dp.host]) - _dp.partitions = pm - - # Write a sched.py and param.py for each partition method - ret = self._create_exp(_dp, ts, graphs, subts) - if not ret: - print("Bin-packing fails for " + dir_leaf) - last_failed = dir_leaf - raise Exception("Failed to partition.") - del(self.out_dir) - if PARAMS['trial'] in dp: - del dp[PARAMS['trial']] - del _dp[PARAMS['trial']] - # just generate one experiment for global - if dp['level'] == 'ALL': - break + for shp in shared_params: + dp['level'] = shp.clustering + _dp['level'] = shp.clustering + + # load in the shared parameters + dp.partitions = shp.partitioning + dp.cluster = shp.clustering + dp.polluters = shp.polluting + dp.job_splitting = shp.splitting + + # Create directory name from relevant parameters + dir_parts = [] + dir_parts.append("sched=%s" % self.scheduler) + dir_parts.append("cluster=%s" % shp.clustering) + dir_parts.append("polluterovh=%s" % shp.polluting) + dir_parts.append("splitting=%s" % shp.splitting) + others = col_map.encode(dp) + if others != "": + dir_parts.append(others) + if trials > 1: + dir_parts.append("trial=%d" % trial) + dir_leaf = "_".join(dir_parts) + dir_path = "%s/%s" % (out_dir, dir_leaf) + + print("Generating %s" % dir_leaf) + + if os.path.exists(dir_path): + if force: + sh.rmtree(dir_path) + else: + print("Skipping existing experiment: '%s'" % dir_path) + continue + + os.mkdir(dir_path) + created_dirs.append(dir_path) + + if trials > 1: + dp[PARAMS['trial']] = trial + _dp[PARAMS['trial']] = trial + self.out_dir = dir_path + + _dp.system = topology.Topology(machines[dp.host]) + _dp.partitions = dp.partitions + _dp.polluters = dp.polluters + _dp.job_splitting = dp.job_splitting + + # Write a sched.py and param.py for each partition method + ret = self._create_exp(_dp, ts, graphs, subts) + if not ret: + print("Bin-packing fails for " + dir_leaf) + last_failed = dir_leaf + raise Exception("Failed to partition.") + del(self.out_dir) + if PARAMS['trial'] in dp: + del dp[PARAMS['trial']] + del _dp[PARAMS['trial']] success = True except Exception, e: for d in created_dirs: sh.rmtree(d) - if not success: - print("Failed to generate experiments. Last failed: %s" % last_failed) + if not success: + print("Failed to generate experiment (%s). Try count = %d" % (last_failed, tries)) def print_help(self): -- cgit v1.2.2