From d169debf732270c9571be6ea6e7d920345bffc33 Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Mon, 8 Apr 2013 15:42:16 -0400 Subject: Added parameter file options for tracers and pre/post experiment commands. --- common.py | 2 +- config/config.py | 5 +- run/experiment.py | 56 +++++-------- run/tracer.py | 52 +++++++++++- run_exps.py | 242 ++++++++++++++++++++++++++++++++++-------------------- 5 files changed, 230 insertions(+), 127 deletions(-) diff --git a/common.py b/common.py index a26ac94..66df7bb 100644 --- a/common.py +++ b/common.py @@ -7,7 +7,7 @@ import sys from collections import defaultdict from textwrap import dedent -def get_executable(prog, hint, optional=False): +def get_executable(prog, hint='unknown', optional=False): '''Search for @prog in system PATH. Print @hint if no binary is found.''' def is_exe(fpath): diff --git a/config/config.py b/config/config.py index 49fd234..3486a40 100644 --- a/config/config.py +++ b/config/config.py @@ -28,7 +28,10 @@ PARAMS = {'sched' : 'scheduler', # Scheduler used by run_exps 'copts' : 'config-options', # Required kernel configuration options 'cycles' : 'clock-frequency', # Frequency run_exps was run with 'tasks' : 'tasks', # Number of tasks - 'trial' : 'trial' # For multiple exps with same config + 'trial' : 'trial', # For multiple exps with same config + 'pre' : 'pre-experiment', # Run before each experiment + 'post' : 'post-experiment', # Run after each experiment + 'trace' : 'tracers' # Tracers to run with an experiment } '''Default values for program options.''' diff --git a/run/experiment.py b/run/experiment.py index 03d6ab6..3dd4866 100644 --- a/run/experiment.py +++ b/run/experiment.py @@ -3,7 +3,6 @@ import time import run.litmus_util as lu import shutil as sh from operator import methodcaller -from run.tracer import SchedTracer, LogTracer, PerfTracer, LinuxTracer, OverheadTracer class ExperimentException(Exception): '''Used to indicate when there are problems with an experiment.''' @@ -32,7 +31,8 @@ class Experiment(object): '''Execute one task-set and save the results. Experiments have unique IDs.''' INTERRUPTED_DIR = ".interrupted" - def __init__(self, name, scheduler, working_dir, finished_dir, proc_entries, executables): + def __init__(self, name, scheduler, working_dir, finished_dir, + proc_entries, executables, tracer_types): '''Run an experiment, optionally wrapped in tracing.''' self.name = name @@ -46,27 +46,17 @@ class Experiment(object): self.__make_dirs() self.__assign_executable_cwds() + self.__setup_tracers(tracer_types) - self.tracers = [] - if SchedTracer.enabled(): - self.log("Enabling sched_trace") - self.tracers.append( SchedTracer(working_dir) ) - if LinuxTracer.enabled(): - self.log("Enabling trace-cmd") - self.tracers.append( LinuxTracer(working_dir) ) - if LogTracer.enabled(): - self.log("Enabling logging") - self.tracers.append( LogTracer(working_dir) ) - if PerfTracer.enabled(): - self.log("Tracking CPU performance counters") - self.tracers.append( PerfTracer(working_dir) ) - - # Overhead trace must be handled seperately, see __run_tasks - if OverheadTracer.enabled(): - self.log("Enabling overhead tracing") - self.overhead_trace = OverheadTracer(working_dir) - else: - self.overhead_trace = None + + def __setup_tracers(self, tracer_types): + tracers = [ t(self.working_dir) for t in tracer_types ] + + self.regular_tracers = [t for t in tracers if not t.is_exact()] + self.exact_tracers = [t for t in tracers if t.is_exact()] + + for t in tracers: + self.log("Enabling %s" % t.get_name()) def __make_dirs(self): interrupted = None @@ -111,11 +101,10 @@ class Experiment(object): raise Exception("Too much time has passed waiting for tasks!") time.sleep(1) - # Overhead tracer must be started right after release or overhead + # Exact tracers (like overheads) must be started right after release or # measurements will be full of irrelevant records - if self.overhead_trace: - self.log("Starting overhead trace") - self.overhead_trace.start_tracing() + self.log("Starting %d released tracers" % len(self.exact_tracers)) + map(methodcaller('start_tracing'), self.exact_tracers) self.log("Releasing %d tasks" % len(self.executables)) released = lu.release_tasks() @@ -145,10 +134,9 @@ class Experiment(object): if not e.wait(): ret = False - # And it must be stopped here for the same reason - if self.overhead_trace: - self.log("Stopping overhead trace") - self.overhead_trace.stop_tracing() + # And these must be stopped here for the same reason + self.log("Stopping exact tracers") + map(methodcaller('stop_tracing'), self.exact_tracers) if not ret: raise ExperimentFailed(self.name) @@ -186,8 +174,8 @@ class Experiment(object): self.log("Switching to %s" % self.scheduler) lu.switch_scheduler(self.scheduler) - self.log("Starting %d tracers" % len(self.tracers)) - map(methodcaller('start_tracing'), self.tracers) + self.log("Starting %d regular tracers" % len(self.regular_tracers)) + map(methodcaller('start_tracing'), self.regular_tracers) self.exec_out = open('%s/exec-out.txt' % self.working_dir, 'w') self.exec_err = open('%s/exec-err.txt' % self.working_dir, 'w') @@ -200,6 +188,6 @@ class Experiment(object): self.exec_out and self.exec_out.close() self.exec_err and self.exec_err.close() - self.log("Stopping tracers") - map(methodcaller('stop_tracing'), self.tracers) + self.log("Stopping regular tracers") + map(methodcaller('stop_tracing'), self.regular_tracers) diff --git a/run/tracer.py b/run/tracer.py index 065797c..6e1d05c 100644 --- a/run/tracer.py +++ b/run/tracer.py @@ -6,10 +6,17 @@ from operator import methodcaller from run.executable.ftcat import FTcat,Executable class Tracer(object): - def __init__(self, name, output_dir): + def __init__(self, name, output_dir, exact=False): self.name = name self.output_dir = output_dir self.bins = [] + self.exact=exact + + def get_name(self): + return self.name + + def is_exact(self): + return self.exact def start_tracing(self): map(methodcaller("execute"), self.bins) @@ -23,7 +30,7 @@ class LinuxTracer(Tracer): LITMUS_EVENTS = "%s/events/litmus" % EVENT_ROOT def __init__(self, output_dir): - super(LinuxTracer, self).__init__("trace-cmd", output_dir) + super(LinuxTracer, self).__init__("Trace-cmd / Kernelshark", output_dir) extra_args = ["record", # "-e", "sched:sched_switch", "-e", "litmus:*", @@ -89,7 +96,7 @@ class OverheadTracer(Tracer): DEVICE_STR = '/dev/litmus/ft_trace0' def __init__(self, output_dir): - super(OverheadTracer, self).__init__("Overhead Trace", output_dir) + super(OverheadTracer, self).__init__("Overhead Trace", output_dir, True) stdout_f = open('{0}/{1}'.format(self.output_dir, conf.FILES['ft_data']), 'w') stderr_f = open('{0}/{1}.stderr.txt'.format(self.output_dir, conf.FILES['ft_data']), 'w') @@ -109,3 +116,42 @@ class PerfTracer(Tracer): @staticmethod def enabled(): return False + + +tracers = {} + +def register_tracer(tracer, name): + tracers[name] = tracer + +def get_tracer_types(names): + error = True # Error if name is not present + errors = [] + + if not names: + # Just return all enabled tracers if none specified + names = tracers.keys() + error = False + + ret = [] + + for name in names: + if name not in tracers: + raise ValueError("Invalid tracer '%s', valid names are: %s" % + (name, tracers.keys())) + + if tracers[name].enabled(): + ret += [ tracers[name] ] + elif error: + errors += ["Tracer '%s' requested, but not enabled." % name] + + if errors: + raise ValueError("Check your kernel compile configuration!\n" + + "\n".join(errors)) + + return ret + +register_tracer(LinuxTracer, "kernelshark") +register_tracer(LogTracer, "log") +register_tracer(SchedTracer, "sched") +register_tracer(OverheadTracer, "overhead") + diff --git a/run_exps.py b/run_exps.py index c51f4c6..6043931 100755 --- a/run_exps.py +++ b/run_exps.py @@ -7,6 +7,7 @@ import os import re import shutil import sys +import run.tracer as trace import traceback from collections import namedtuple @@ -15,6 +16,12 @@ from run.executable.executable import Executable from run.experiment import Experiment,ExperimentDone from run.proc_entry import ProcEntry +'''Customizable experiment parameters''' +ExpParams = namedtuple('ExpParams', ['scheduler', 'duration', 'tracers', + 'kernel', 'config_options']) +'''Comparison of requested versus actual kernel compile parameter value''' +ConfigResult = namedtuple('ConfigResult', ['param', 'wanted', 'actual']) + class InvalidKernel(Exception): def __init__(self, kernel): self.kernel = kernel @@ -22,7 +29,7 @@ class InvalidKernel(Exception): def __str__(self): return "Kernel name does not match '%s'." % self.kernel -ConfigResult = namedtuple('ConfigResult', ['param', 'wanted', 'actual']) + class InvalidConfig(Exception): def __init__(self, results): self.results = results @@ -38,6 +45,7 @@ class InvalidConfig(Exception): return "Invalid kernel configuration " +\ "(ignore configuration with -i option).\n" + "\n".join(messages) + def parse_args(): parser = OptionParser("usage: %prog [options] [sched_file]... [exp_dir]...") @@ -92,6 +100,7 @@ def convert_data(data): return {'proc' : procs, 'spin' : spins} + def fix_paths(schedule, exp_dir, sched_file): '''Replace relative paths of command line arguments with absolute ones.''' for (idx, (spin, args)) in enumerate(schedule['spin']): @@ -106,97 +115,24 @@ def fix_paths(schedule, exp_dir, sched_file): schedule['spin'][idx] = (spin, args) -def verify_environment(kernel, copts): - if kernel and not com.uname_matches(kernel): - raise InvalidKernel(kernel) - - if copts: - results = [] - for param, wanted in copts.iteritems(): - try: - actual = com.get_config_option(param) - except IOError: - actual = None - if not str(wanted) == str(actual): - results += [ConfigResult(param, wanted, actual)] - - if results: - raise InvalidConfig(results) - -def load_experiment(sched_file, scheduler, duration, - param_file, out_dir, ignore, jabber): - if not os.path.isfile(sched_file): - raise IOError("Cannot find schedule file: %s" % sched_file) - - dir_name, fname = os.path.split(sched_file) - exp_name = os.path.split(dir_name)[1] + "/" + fname - - params = {} - kernel = copts = "" - - param_file = param_file or \ - "%s/%s" % (dir_name, conf.DEFAULTS['params_file']) - - if os.path.isfile(param_file): - params = com.load_params(param_file) - scheduler = scheduler or params[conf.PARAMS['sched']] - duration = duration or params[conf.PARAMS['dur']] - - # Experiments can specify required kernel name - if conf.PARAMS['kernel'] in params: - kernel = params[conf.PARAMS['kernel']] - # Or required config options - if conf.PARAMS['copts'] in params: - copts = params[conf.PARAMS['copts']] - - duration = duration or conf.DEFAULTS['duration'] - - if not scheduler: - raise IOError("Parameter scheduler not specified in %s" % (param_file)) - - # Parse schedule file's intentions - schedule = load_schedule(sched_file) - work_dir = "%s/tmp" % dir_name - - fix_paths(schedule, os.path.split(sched_file)[0], sched_file) - - if not ignore: - verify_environment(kernel, copts) - - run_exp(exp_name, schedule, scheduler, kernel, duration, work_dir, out_dir) - - if jabber: - jabber.send("Completed '%s'" % exp_name) - - # Save parameters used to run experiment in out_dir - out_params = dict(params.items() + - [(conf.PARAMS['sched'], scheduler), - (conf.PARAMS['tasks'], len(schedule['spin'])), - (conf.PARAMS['dur'], duration)]) - # Feather-trace clock frequency saved for accurate overhead parsing - ft_freq = com.ft_freq() - if ft_freq: - out_params[conf.PARAMS['cycles']] = ft_freq - - with open("%s/%s" % (out_dir, conf.DEFAULTS['params_file']), 'w') as f: - f.write(str(out_params)) - -def load_schedule(fname): +def load_schedule(name, fname, duration): + '''Turn schedule file @fname into ProcEntry's and Executable's which execute + for @duration time.''' with open(fname, 'r') as f: data = f.read().strip() try: schedule = eval(data) except: schedule = convert_data(data) - return schedule + # Make paths relative to the file's directory + fix_paths(schedule, os.path.split(fname)[0], fname) -def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir): proc_entries = [] executables = [] - # Parse values for proc entries + # Create proc entries for entry_conf in schedule['proc']: path = entry_conf[0] data = entry_conf[1] @@ -206,7 +142,7 @@ def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir): proc_entries += [ProcEntry(path, data)] - # Parse spinners + # Create executables for spin_conf in schedule['spin']: if isinstance(spin_conf, str): # Just a string defaults to default spin @@ -217,9 +153,6 @@ def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir): raise IOError("Invalid spin conf %s: %s" % (spin_conf, name)) (spin, args) = (spin_conf[0], spin_conf[1]) - # if not conf.BINS[spin]: - # raise IndexError("No knowledge of program %s: %s" % (spin, name)) - real_spin = com.get_executable(spin, "") real_args = args.split() if re.match(".*spin", real_spin): @@ -230,21 +163,154 @@ def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir): executables += [Executable(real_spin, real_args)] - exp = Experiment(name, scheduler, work_dir, out_dir, - proc_entries, executables) + return proc_entries, executables + + +def verify_environment(exp_params): + if exp_params.kernel and not com.uname_matches(exp_params.kernel): + raise InvalidKernel(exp_params.kernel) + + if exp_params.config_options: + results = [] + for param, wanted in exp_params.config_options.iteritems(): + try: + actual = com.get_config_option(param) + except IOError: + actual = None + if not str(wanted) == str(actual): + results += [ConfigResult(param, wanted, actual)] + + if results: + raise InvalidConfig(results) + + +def run_parameter(exp_dir, out_dir, params, param_name): + '''Run an executable (arguments optional) specified as a configurable + @param_name in @params.''' + if conf.PARAMS[param_name] not in params: + return + + script_params = params[conf.PARAMS[param_name]] + + # Split into arguments and program name + if type(script_params) != type([]): + script_params = [script_params] + script_name = script_params.pop(0) + + cwd_name = "%s/%s" % (exp_dir, script_name) + if os.path.isfile(cwd_name): + script = cwd_name + else: + script = com.get_executable(script_name, optional=True) + + if not script: + raise Exception("Cannot find executable %s-script: %s" % + (param_name, script_name)) + + out = open('%s/%s-out.txt' % (out_dir, param_name), 'w') + prog = Executable(script, script_params, + stderr_file=out, stdout_file=out) + prog.cwd = out_dir + + prog.execute() + prog.wait() + + out.close() + + +def get_exp_params(cmd_scheduler, cmd_duration, file_params): + kernel = copts = "" + + scheduler = cmd_scheduler or file_params[conf.PARAMS['sched']] + duration = cmd_duration or file_params[conf.PARAMS['dur']] or\ + conf.DEFAULTS['duration'] + + # Experiments can specify required kernel name + if conf.PARAMS['kernel'] in file_params: + kernel = file_params[conf.PARAMS['kernel']] + + # Or required config options + if conf.PARAMS['copts'] in file_params: + copts = file_params[conf.PARAMS['copts']] + + # Or required tracers + requested = [] + if conf.PARAMS['trace'] in file_params: + requested = file_params[conf.PARAMS['trace']] + tracers = trace.get_tracer_types(requested) + + # But only these two are mandatory + if not scheduler: + raise IOError("No scheduler found in param file!") + if not duration: + raise IOError("No duration found in param file!") + + return ExpParams(scheduler=scheduler, kernel=kernel, duration=duration, + config_options=copts, tracers=tracers) + + +def load_experiment(sched_file, cmd_scheduler, cmd_duration, + param_file, out_dir, ignore, jabber): + '''Load and parse data from files and run result.''' + if not os.path.isfile(sched_file): + raise IOError("Cannot find schedule file: %s" % sched_file) + + dir_name, fname = os.path.split(sched_file) + exp_name = os.path.split(dir_name)[1] + "/" + fname + work_dir = "%s/tmp" % dir_name + + # Load parameter file + param_file = param_file or \ + "%s/%s" % (dir_name, conf.DEFAULTS['params_file']) + if os.path.isfile(param_file): + file_params = com.load_params(param_file) + else: + file_params = {} + + exp_params = get_exp_params(cmd_scheduler, cmd_duration, file_params) + procs, execs = load_schedule(exp_name, sched_file, exp_params.duration) + + exp = Experiment(exp_name, exp_params.scheduler, work_dir, out_dir, + procs, execs, exp_params.tracers) + + if not ignore: + verify_environment(exp_params) + + run_parameter(dir_name, work_dir, file_params, 'pre') exp.run_exp() + run_parameter(dir_name, out_dir, file_params, 'post') + + if jabber: + jabber.send("Completed '%s'" % exp_name) + + # Save parameters used to run experiment in out_dir + out_params = dict(file_params.items() + + [(conf.PARAMS['sched'], exp_params.scheduler), + (conf.PARAMS['tasks'], len(execs)), + (conf.PARAMS['dur'], exp_params.duration)]) + + # Feather-trace clock frequency saved for accurate overhead parsing + ft_freq = com.ft_freq() + if ft_freq: + out_params[conf.PARAMS['cycles']] = ft_freq + + with open("%s/%s" % (out_dir, conf.DEFAULTS['params_file']), 'w') as f: + f.write(str(out_params)) + + def setup_jabber(target): try: from run.jabber import Jabber return Jabber(target) except ImportError: - sys.stderr.write("Failed to import jabber. Is python-xmpppy "+\ - "installed?\nDisabling Jabber messaging.\n") + sys.stderr.write("Failed to import jabber, disabling messages. " + + "Is python-xmpp installed?") return None + def main(): opts, args = parse_args() -- cgit v1.2.2