Read locations of binary files from path instead of config.py.

8 files changed, 522 insertions, 0 deletions

diff --git a/run/__init__.py b/run/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/run/__init__.py
diff --git a/run/executable/__init__.py b/run/executable/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/run/executable/__init__.py
diff --git a/run/executable/executable.py b/run/executable/executable.py
new file mode 100644
index 0000000..628f711
--- /dev/null
+++ b/run/executable/executable.py
@@ -0,0 +1,77 @@
+import sys
+import subprocess
+import signal
+from ..litmus_util import is_executable
+
+class Executable(object):
+    '''Parent object that represents an executable for use in task-sets.'''
+
+    def __init__(self, exec_file, extra_args=None, stdout_file = None, stderr_file = None):
+        self.exec_file = exec_file
+        self.cwd = None
+        self.stdout_file = stdout_file
+        self.stderr_file = stderr_file
+        self.sp = None
+
+        if extra_args is None:
+            self.extra_args = None
+        else:
+            self.extra_args = list(extra_args) # make a duplicate
+
+        if not is_executable(self.exec_file):
+            raise Exception("Not executable ? : %s" % self.exec_file)
+
+    def __del__(self):
+        # Try and clean up
+        if self.stdout_file is not None:
+            self.stdout_file.close()
+        if self.stderr_file is not None:
+            self.stderr_file.close()
+
+        if self.sp is not None:
+            try:
+                self.sp.terminate()
+            except OSError as e:
+                if e.errno == 3:
+                    pass # no such process (already killed), okay
+                else:
+                    raise e
+
+    def __get_full_command(self):
+        full_command = [self.exec_file]
+        if self.extra_args is not None:
+            full_command += self.extra_args
+        return full_command
+
+    def __str__(self):
+        return " ".join(self.__get_full_command())
+
+    def execute(self):
+        '''Execute the binary.'''
+        full_command = self.__get_full_command()
+        self.sp = subprocess.Popen(full_command, stdout=self.stdout_file,
+                stderr=self.stderr_file, cwd=self.cwd)
+
+    def kill(self):
+        self.sp.kill()
+
+    def interrupt(self):
+        self.sp.send_signal(signal.SIGINT)
+
+    def terminate(self):
+        '''Send the terminate signal to the binary.'''
+        self.sp.terminate()
+
+    def wait(self):
+        '''Wait until the executable is finished, checking return code.
+
+        If the exit status is non-zero, raise an exception.
+
+        '''
+
+        self.sp.wait()
+        if self.sp.returncode != 0:
+            print >>sys.stderr, "Non-zero return: %s %s" % (self.exec_file, " ".join(self.extra_args))
+            return 0
+        else:
+            return 1
diff --git a/run/executable/ftcat.py b/run/executable/ftcat.py
new file mode 100644
index 0000000..5da8fa7
--- /dev/null
+++ b/run/executable/ftcat.py
@@ -0,0 +1,33 @@
+import os
+import stat
+
+from executable import Executable
+
+class FTcat(Executable):
+    '''Used to wrap the ftcat binary in the Experiment object.'''
+
+    def __init__(self, ft_cat_bin, stdout_file, stderr_file, dev, events, cpu=None):
+        '''Extends the Executable initializer method with ftcat attributes.'''
+
+        # hack to run FTCat at higher priority
+        chrt_bin = '/usr/bin/chrt'
+
+        super(FTcat, self).__init__(chrt_bin)
+        self.stdout_file = stdout_file
+        self.stderr_file = stderr_file
+
+        mode = os.stat(dev)[stat.ST_MODE]
+        if not mode & stat.S_IFCHR:
+            raise Exception("%s is not a character device" % dev)
+
+        if events is None:
+            raise Exception('No events!')
+
+        # hack to run FTCat at higher priority
+        self.extra_args = ['-f', '40']
+        if cpu is not None:
+            # and bind it to a CPU
+            self.extra_args.extend(['/usr/bin/taskset', '-c', str(cpu)])
+        events_str_arr = map(str, events)
+        self.extra_args.extend([ft_cat_bin, dev] + events_str_arr)
+
diff --git a/run/experiment.py b/run/experiment.py
new file mode 100644
index 0000000..4bd47c6
--- /dev/null
+++ b/run/experiment.py
@@ -0,0 +1,209 @@
+import os
+import time
+import litmus_util
+from operator import methodcaller
+from tracer import SchedTracer, LogTracer, PerfTracer, LinuxTracer, OverheadTracer
+
+class ExperimentException(Exception):
+    '''Used to indicate when there are problems with an experiment.'''
+    def __init__(self, name):
+        self.name = name
+
+
+class ExperimentDone(ExperimentException):
+    '''Raised when an experiment looks like it's been run already.'''
+    def __str__(self):
+        return "Experiment finished already: %d" % self.name
+
+
+class ExperimentInterrupted(ExperimentException):
+    '''Raised when an experiment appears to be interrupted (partial results).'''
+    def __str__(self):
+        return "Experiment was interrupted in progress: %d" % self.name
+
+
+class ExperimentFailed(ExperimentException):
+    def __str__(self):
+        return "Experiment failed during execution: %d" % self.name
+
+
+class Experiment(object):
+    '''Execute one task-set and save the results. Experiments have unique IDs.'''
+    INTERRUPTED_DIR = ".interrupted"
+
+    def __init__(self, name, scheduler, working_dir, finished_dir, proc_entries, executables):
+        '''Run an experiment, optionally wrapped in tracing.'''
+
+        self.name = name
+        self.scheduler = scheduler
+        self.working_dir  = working_dir
+        self.finished_dir = finished_dir
+        self.proc_entries = proc_entries
+        self.executables  = executables
+        self.exec_out = None
+        self.exec_err = None
+
+        self.__make_dirs()
+        self.__assign_executable_cwds()
+
+        self.tracers = []
+        if SchedTracer.enabled():
+            self.log("Enabling sched_trace")
+            self.tracers.append( SchedTracer(working_dir) )
+        if LinuxTracer.enabled():
+            self.log("Enabling trace-cmd / ftrace / kernelshark")
+            self.tracers.append( LinuxTracer(working_dir) )
+        if LogTracer.enabled():
+            self.log("Enabling logging")
+            self.tracers.append( LogTracer(working_dir) )
+        if PerfTracer.enabled():
+            self.log("Tracking CPU performance counters")
+            self.tracers.append( PerfTracer(working_dir) )
+
+        # Overhead trace must be handled seperately, see __run_tasks
+        if OverheadTracer.enabled():
+            self.log("Enabling overhead tracing")
+            self.overhead_trace = OverheadTracer(working_dir)
+        else:
+            self.overhead_trace = None
+
+    def __make_dirs(self):
+        interrupted = None
+
+        if os.path.exists(self.finished_dir):
+            raise ExperimentDone(self.name)
+
+        if os.path.exists(self.working_dir):
+            self.log("Found interrupted experiment, saving in %s" %
+                     Experiment.INTERRUPTED_DIR)
+            interrupted = "%s/%s" % (os.path.split(self.working_dir)[0],
+                                     Experiment.INTERRUPTED_DIR)
+            os.rename(self.working_dir, interrupted)
+
+        os.mkdir(self.working_dir)
+
+        if interrupted:
+            os.rename(interrupted, "%s/%s" % (self.working_dir,
+                                              os.path.split(interrupted)[1]))
+
+    def __assign_executable_cwds(self):
+        def assign_cwd(executable):
+            executable.cwd = self.working_dir
+        map(assign_cwd, self.executables)
+
+    def __run_tasks(self):
+        exec_pause = 0.3
+        self.log("Starting the programs over ({0} seconds)".format(
+            len(self.executables) * exec_pause))
+        for e in self.executables:
+            try:
+                e.execute()
+            except:
+                raise Exception("Executable failed: %s" % e)
+            time.sleep(exec_pause)
+
+        sleep_time = len(self.executables) / litmus_util.num_cpus()
+        self.log("Sleeping for %d seconds before release" % sleep_time)
+        time.sleep(sleep_time)
+
+        # Overhead tracer must be started right after release or overhead
+        # measurements will be full of irrelevant records
+        if self.overhead_trace:
+            self.log("Starting overhead trace")
+            self.overhead_trace.start_tracing()
+
+        self.log("Releasing %d tasks" % len(self.executables))
+        released = litmus_util.release_tasks()
+
+        ret = True
+        if released != len(self.executables):
+            # Some tasks failed to release, kill all tasks and fail
+            # Need to re-release non-released tasks before we can kill them though
+            self.log("Failed to release {} tasks! Re-releasing and killing".format(
+                len(self.executables) - released, len(self.executables)))
+
+            time.sleep(5)
+
+            released = litmus_util.release_tasks()
+
+            self.log("Re-released %d tasks" % released)
+
+            time.sleep(5)
+
+            self.log("Killing all tasks")
+            map(methodcaller('kill'), self.executables)
+
+            ret = False
+
+        self.log("Waiting for program to finish...")
+        for e in self.executables:
+            if not e.wait():
+                ret = False
+
+        # And it must be stopped here for the same reason
+        if self.overhead_trace:
+            self.log("Stopping overhead trace")
+            self.overhead_trace.stop_tracing()
+
+        if not ret:
+            raise ExperimentFailed(self.name)
+
+    def __save_results(self):
+        os.rename(self.working_dir, self.finished_dir)
+
+    def log(self, msg):
+        print "[Exp %s]: %s" % (self.name, msg)
+
+    def run_exp(self):
+        succ = False
+        try:
+            self.setup()
+
+            try:
+                self.__run_tasks()
+                self.log("Saving results in %s" % self.finished_dir)
+                succ = True
+            finally:
+                self.teardown()
+        finally:
+            self.log("Switching to Linux scheduler")
+            litmus_util.switch_scheduler("Linux")
+
+        if succ:
+            self.__save_results()
+            self.log("Experiment done!")
+
+
+    def setup(self):
+        self.log("Writing %d proc entries" % len(self.proc_entries))
+        map(methodcaller('write_proc'), self.proc_entries)
+
+        if len(self.proc_entries):
+            time.sleep(2)
+
+        self.log("Switching to %s" % self.scheduler)
+        litmus_util.switch_scheduler(self.scheduler)
+
+        self.log("Starting %d tracers" % len(self.tracers))
+        map(methodcaller('start_tracing'), self.tracers)
+
+        self.exec_out = open('%s/exec-out.txt' % self.working_dir, 'w')
+        self.exec_err = open('%s/exec-err.txt' % self.working_dir, 'w')
+        def set_out(executable):
+            executable.stdout_file = self.exec_out
+            executable.stderr_file = self.exec_err
+        map(set_out, self.executables)
+
+        time.sleep(4)
+
+    def teardown(self):
+        self.exec_out and self.exec_out.close()
+        self.exec_err and self.exec_err.close()
+
+        sleep_time = 10
+        self.log("Sleeping %d seconds to allow buffer flushing" % sleep_time)
+        time.sleep(sleep_time)
+
+        self.log("Stopping tracers")
+        map(methodcaller('stop_tracing'), self.tracers)
+
diff --git a/run/litmus_util.py b/run/litmus_util.py
new file mode 100644
index 0000000..fb2b341
--- /dev/null
+++ b/run/litmus_util.py
@@ -0,0 +1,79 @@
+import re
+import time
+import subprocess
+import os
+import stat
+import config.config as conf
+
+def num_cpus():
+    '''Return the number of CPUs in the system.'''
+
+    lnx_re = re.compile(r'^(processor|online)')
+    cpus = 0
+
+    with open('/proc/cpuinfo', 'r') as f:
+        for line in f:
+            if lnx_re.match(line):
+                cpus += 1
+    return cpus
+
+def cpu_freq():
+    '''
+    The frequency (in MHz) of the CPU.
+    '''
+    reg = re.compile(r'^cpu MHz\s*:\s*(\d+)', re.M)
+    with open('/proc/cpuinfo', 'r') as f:
+        data = f.read()
+
+    match = re.search(reg, data)
+    if not match:
+        raise Exception("Cannot parse CPU frequency!")
+    return int(match.group(1))
+
+def switch_scheduler(switch_to_in):
+    '''Switch the scheduler to whatever is passed in.
+
+    This methods sleeps for two seconds to give Linux the chance to execute
+    schedule switching code. Raises an exception if the switch does not work.
+
+    '''
+
+    switch_to = str(switch_to_in).strip()
+
+    with open('/proc/litmus/active_plugin', 'w') as active_plugin:
+        subprocess.Popen(["echo", switch_to], stdout=active_plugin)
+
+    # it takes a bit to do the switch, sleep an arbitrary amount of time
+    time.sleep(2)
+
+    with open('/proc/litmus/active_plugin', 'r') as active_plugin:
+        cur_plugin = active_plugin.read().strip()
+
+    if switch_to != cur_plugin:
+        raise Exception("Could not switch to plugin: %s" % switch_to)
+
+def uname_matches(reg):
+    data = subprocess.check_output(["uname", "-r"])
+    return bool( re.match(reg, data) )
+
+def is_executable(fname):
+    '''Return whether the file passed in is executable'''
+    mode = os.stat(fname)[stat.ST_MODE]
+    return mode & stat.S_IXUSR and mode & stat.S_IRUSR
+
+def is_device(dev):
+    if not os.path.exists(dev):
+        return False
+    mode = os.stat(dev)[stat.ST_MODE]
+    return not (not mode & stat.S_IFCHR)
+
+def release_tasks():
+
+    try:
+        data = subprocess.check_output([conf.BINS['release']])
+    except subprocess.CalledProcessError:
+        raise Exception('Something went wrong in release_ts')
+
+    released = re.findall(r"([0-9]+) real-time", data)[0]
+
+    return int(released)
diff --git a/run/proc_entry.py b/run/proc_entry.py
new file mode 100644
index 0000000..0b7f9ce
--- /dev/null
+++ b/run/proc_entry.py
@@ -0,0 +1,12 @@
+import os
+
+class ProcEntry(object):
+    def __init__(self, proc, data):
+        self.proc = proc
+        self.data = data
+
+    def write_proc(self):
+        if not os.path.exists(self.proc):
+            raise Exception("Invalid proc entry %s" % self.proc)
+        with open(self.proc, 'w') as entry:
+            entry.write(self.data)
diff --git a/run/tracer.py b/run/tracer.py
new file mode 100644
index 0000000..5d00e86
--- /dev/null
+++ b/run/tracer.py
@@ -0,0 +1,112 @@
+import litmus_util
+import os
+import config.config as conf
+
+from operator import methodcaller
+from executable.ftcat import FTcat,Executable
+
+
+class Tracer(object):
+    def __init__(self, name, output_dir):
+        self.name = name
+        self.output_dir = output_dir
+        self.bins = []
+
+    def start_tracing(self):
+        map(methodcaller("execute"), self.bins)
+
+    def stop_tracing(self):
+        map(methodcaller('terminate'), self.bins)
+        map(methodcaller('wait'), self.bins)
+
+
+class LinuxTracer(Tracer):
+    EVENT_ROOT = "/sys/kernel/debug/tracing"
+    LITMUS_EVENTS = "%s/events/litmus" % EVENT_ROOT
+
+    def __init__(self, output_dir):
+        super(LinuxTracer, self).__init__("trace-cmd", output_dir)
+
+        extra_args = ["record", # "-e", "sched:sched_switch",
+                      "-e", "litmus:*",
+                      "-o", "%s/%s" % (output_dir, conf.FILES['linux_data'])]
+        stdout = open('%s/trace-cmd-stdout.txt' % self.output_dir, 'w')
+        stderr = open('%s/trace-cmd-stderr.txt' % self.output_dir, 'w')
+
+        execute = Executable(conf.BINS['trace-cmd'], extra_args, stdout, stderr)
+        execute.cwd = output_dir
+        self.bins.append(execute)
+
+    @staticmethod
+    def enabled():
+        return os.path.exists(LinuxTracer.LITMUS_EVENTS)
+
+    def stop_tracing(self):
+        map(methodcaller('interrupt'), self.bins)
+        map(methodcaller('wait'), self.bins)
+
+
+class LogTracer(Tracer):
+    DEVICE_STR = '/dev/litmus/log'
+
+    def __init__(self, output_dir):
+        super(LogTracer, self).__init__("Logger", output_dir)
+
+        out_file = open("%s/%s" % (self.output_dir, conf.FILES['log_data']), 'w')
+
+        cat = (Executable("/bin/cat", [LogTracer.DEVICE_STR]))
+        cat.stdout_file = out_file
+
+        self.bins.append(cat)
+
+    @staticmethod
+    def enabled():
+        return litmus_util.is_device(LogTracer.DEVICE_STR)
+
+
+class SchedTracer(Tracer):
+    DEVICE_STR = '/dev/litmus/sched_trace'
+
+    def __init__(self, output_dir):
+        super(SchedTracer, self).__init__("Sched Trace", output_dir)
+
+        if SchedTracer.enabled():
+            for cpu in range(litmus_util.num_cpus()):
+                # Executable will close the stdout/stderr files
+                stdout_f = open('%s/st-%d.bin' % (self.output_dir, cpu), 'w')
+                stderr_f = open('%s/st-%d-stderr.txt' % (self.output_dir, cpu), 'w')
+                dev = '{0}{1}'.format(SchedTracer.DEVICE_STR, cpu)
+                ftc = FTcat(conf.BINS['ftcat'], stdout_f, stderr_f, dev, conf.SCHED_EVENTS, cpu=cpu)
+
+                self.bins.append(ftc)
+
+    @staticmethod
+    def enabled():
+                return litmus_util.is_device("%s%d" % (SchedTracer.DEVICE_STR, 0))
+
+
+class OverheadTracer(Tracer):
+    DEVICE_STR = '/dev/litmus/ft_trace0'
+
+    def __init__(self, output_dir):
+        super(OverheadTracer, self).__init__("Overhead Trace", output_dir)
+
+        stdout_f = open('{0}/{1}'.format(self.output_dir, conf.FILES['ft_data']), 'w')
+        stderr_f = open('{0}/{1}.stderr.txt'.format(self.output_dir, conf.FILES['ft_data']), 'w')
+        ftc = FTcat(conf.BINS['ftcat'], stdout_f, stderr_f,
+                OverheadTracer.DEVICE_STR, conf.OVH_ALL_EVENTS)
+
+        self.bins.append(ftc)
+
+    @staticmethod
+    def enabled():
+                return litmus_util.is_device(OverheadTracer.DEVICE_STR)
+
+
+class PerfTracer(Tracer):
+    def __init__(self, output_dir):
+        super(PerfTracer, self).__init__("CPU perf counters", output_dir)
+
+    @staticmethod
+    def enabled():
+        return False