Created run_exps.py script.

Currently poorly documented.

8 files changed, 450 insertions, 0 deletions

diff --git a/experiment/__init__.py b/experiment/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/experiment/__init__.py
diff --git a/experiment/executable/__init__.py b/experiment/executable/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/experiment/executable/__init__.py
diff --git a/experiment/executable/executable.py b/experiment/executable/executable.py
new file mode 100644
index 0000000..6697a8d
--- /dev/null
+++ b/experiment/executable/executable.py
@@ -0,0 +1,71 @@
+import sys
+import subprocess
+import signal
+from ..litmus_util import is_executable
+
+class Executable(object):
+    """Parent object that represents an executable for use in task-sets."""
+
+    def __init__(self, exec_file, extra_args=None, stdout_file = None, stderr_file = None):
+        self.exec_file = exec_file
+        self.cwd = None
+        self.stdout_file = stdout_file
+        self.stderr_file = stderr_file
+        self.sp = None
+
+        if extra_args is None:
+            self.extra_args = None
+        else:
+            self.extra_args = list(extra_args) # make a duplicate
+
+        if not is_executable(self.exec_file):
+            raise Exception("Not executable ? : %s" % self.exec_file)
+
+    def __del__(self):
+        # Try and clean up
+        if self.stdout_file is not None:
+            self.stdout_file.close()
+        if self.stderr_file is not None:
+            self.stderr_file.close()
+
+        if self.sp is not None:
+            try:
+                self.sp.terminate()
+            except OSError as e:
+                if e.errno == 3:
+                    pass # no such process (already killed), okay
+                else:
+                    raise e
+
+    def __get_full_command(self):
+        full_command = [self.exec_file]
+        if self.extra_args is not None:
+            full_command += self.extra_args
+        return full_command
+
+    def execute(self):
+        """Execute the binary."""
+        full_command = self.__get_full_command()
+        self.sp = subprocess.Popen(full_command, stdout=self.stdout_file,
+                stderr=self.stderr_file, cwd=self.cwd)
+
+    def kill(self):
+        self.sp.kill()
+
+    def interrupt(self):
+        self.sp.send_signal(signal.SIGINT)
+
+    def terminate(self):
+        """Send the terminate signal to the binary."""
+        self.sp.terminate()
+
+    def wait(self):
+        """Wait until the executable is finished, checking return code.
+
+        If the exit status is non-zero, raise an exception.
+
+        """
+
+        self.sp.wait()
+        if self.sp.returncode != 0:
+            print >>sys.stderr, "Non-zero return: %s %s" % (self.exec_file, self.extra_args)
diff --git a/experiment/executable/ftcat.py b/experiment/executable/ftcat.py
new file mode 100644
index 0000000..9966312
--- /dev/null
+++ b/experiment/executable/ftcat.py
@@ -0,0 +1,33 @@
+import os
+import stat
+
+from executable import Executable
+
+class FTcat(Executable):
+    """Used to wrap the ftcat binary in the Experiment object."""
+
+    def __init__(self, ft_cat_bin, stdout_file, stderr_file, dev, events, cpu=None):
+        """Extends the Executable initializer method with ftcat attributes."""
+
+        # hack to run FTCat at higher priority
+        chrt_bin = '/usr/bin/chrt'
+
+        super(FTcat, self).__init__(chrt_bin)
+        self.stdout_file = stdout_file
+        self.stderr_file = stderr_file
+
+        mode = os.stat(dev)[stat.ST_MODE]
+        if not mode & stat.S_IFCHR:
+            raise Exception("%s is not a character device" % dev)
+
+        if events is None:
+            raise Exception('No events!')
+
+        # hack to run FTCat at higher priority
+        self.extra_args = ['-f', '40']
+        if cpu is not None:
+            # and bind it to a CPU
+            self.extra_args.extend(['/usr/bin/taskset', '-c', str(cpu)])
+        events_str_arr = map(str, events)
+        self.extra_args.extend([ft_cat_bin, dev] + events_str_arr)
+
diff --git a/experiment/experiment.py b/experiment/experiment.py
new file mode 100644
index 0000000..29e6bd7
--- /dev/null
+++ b/experiment/experiment.py
@@ -0,0 +1,153 @@
+import os
+import time
+import litmus_util
+from operator import methodcaller
+from tracer import SchedTracer, LogTracer, PerfTracer, LinuxTracer, OverheadTracer
+
+class ExperimentException(Exception):
+    """Used to indicate when there are problems with an experiment."""
+    def __init__(self, name):
+        self.name = name
+
+        
+class ExperimentDone(ExperimentException):
+    """Raised when an experiment looks like it's been run already."""
+    def __str__(self):
+        return "Experiment finished already: %d" % self.name
+
+    
+class ExperimentInterrupted(ExperimentException):
+    """Raised when an experiment appears to be interrupted (partial results)."""
+    def __str__(self):
+        return "Experiment was interrupted in progress: %d" % self.name
+
+    
+class ExperimentFailed(ExperimentException):
+    def __str__(self):
+        return "Experiment failed during execution: %d" % self.name
+
+    
+class Experiment(object):
+    """Execute one task-set and save the results. Experiments have unique IDs."""
+
+    def __init__(self, name, scheduler, working_dir, finished_dir, proc_entries, executables):
+        """Run an experiment, optionally wrapped in tracing."""
+
+        self.name = name
+        self.scheduler = scheduler
+        self.working_dir  = working_dir
+        self.finished_dir = finished_dir
+        self.proc_entries = proc_entries
+        self.executables  = executables
+
+        self.__make_dirs()
+        self.__assign_executable_cwds()
+
+        self.tracers = []
+        if SchedTracer.enabled():
+            self.tracers.append( SchedTracer(working_dir) )
+        if LinuxTracer.enabled():
+            self.tracers.append( LinuxTracer(working_dir) )
+        if LogTracer.enabled():
+            self.tracers.append( LogTracer(working_dir) )
+        if PerfTracer.enabled():
+            self.tracers.append( PerfTracer(working_dir) )
+
+        # Overhead trace must be handled seperately, see __run_tasks
+        if OverheadTracer.enabled():
+            self.overhead_trace = OverheadTracer(working_dir)
+        else:
+            self.overhead_trace = None
+
+    def __make_dirs(self):
+        if os.path.exists(self.finished_dir):
+            raise ExperimentDone(self.name)
+        if os.path.exists(self.working_dir):
+            raise ExperimentInterrupted(self.name)
+        
+        os.mkdir(self.working_dir)
+
+    def __assign_executable_cwds(self):
+        def assign_cwd(executable):
+            executable.cwd = self.working_dir
+        map(assign_cwd, self.executables)
+
+    def __run_tasks(self):
+        exec_pause = 0.3
+        self.log("Starting the program in ({0} seconds)".format(
+            len(self.executables) * exec_pause))
+        for e in self.executables:
+            e.execute()
+            time.sleep(exec_pause)
+
+        sleep_time = 2
+        self.log("Sleeping for %d seconds before release" % sleep_time)
+        time.sleep(sleep_time)
+
+        # Overhead tracer must be started right after release or overhead
+        # measurements will be full of irrelevant records
+        if self.overhead_trace:
+            self.log("Starting overhead trace")
+            self.overhead_trace.start_tracing()
+
+        released = litmus_util.release_tasks()
+
+        ret = True
+        if released != len(self.executables):
+            self.log("Failed to release %d tasks! Re-releasing and killing".format(
+                len(self.experiments) - released))
+            
+            time.sleep(10)
+            litmus_util.release_tasks()
+
+            time.sleep(20)
+            map(methodcaller('kill'), self.executables)
+
+            ret = False
+
+        self.log("Waiting for program to finish...")
+        map(methodcaller('wait'), self.executables)
+
+        # And it must be stopped here for the same reason
+        if self.overhead_trace:
+            self.log("Stopping overhead trace")
+            self.overhead_trace.stop_tracing()
+
+        if not ret:
+            raise ExperimentFailed(self.name)
+
+    def __save_results(self):
+        os.rename(self.working_dir, self.finished_dir)
+
+    def log(self, msg):
+        print "[Exp %s]: %s" % (self.name, msg)
+
+    def run_exp(self):
+        self.setup()
+        self.__run_tasks()
+        self.teardown()
+
+    def setup(self):
+        self.log("Switching to %s" % self.scheduler)
+        litmus_util.switch_scheduler(self.scheduler)
+
+        self.log("Writing %d proc entries" % len(self.proc_entries))
+        map(methodcaller('write_proc'), self.proc_entries)
+        
+        self.log("Starting %d tracers" % len(self.tracers))
+        map(methodcaller('start_tracing'), self.tracers)
+
+    def teardown(self):
+        sleep_time = 5
+        self.log("Sleeping %d seconds to allow buffer flushing" % sleep_time)
+        time.sleep(sleep_time)
+
+        self.log("Stopping tracers")
+        map(methodcaller('stop_tracing'), self.tracers)
+
+        self.log("Switching to Linux scheduler")
+        litmus_util.switch_scheduler("Linux")
+
+        self.log("Saving results in %s" % self.finished_dir)
+        self.__save_results()
+        self.log("Experiment done!")
diff --git a/experiment/litmus_util.py b/experiment/litmus_util.py
new file mode 100644
index 0000000..114f4c9
--- /dev/null
+++ b/experiment/litmus_util.py
@@ -0,0 +1,63 @@
+import re
+import time
+import subprocess
+import os
+import stat
+import config.config as conf
+
+def num_cpus():
+    """Return the number of CPUs in the system."""
+
+    lnx_re = re.compile(r'^(processor|online)')
+    cpus = 0
+
+    with open('/proc/cpuinfo', 'r') as f:
+        for line in f:
+            if lnx_re.match(line):
+                cpus += 1
+    return cpus
+
+def switch_scheduler(switch_to_in):
+    """Switch the scheduler to whatever is passed in.
+
+    This methods sleeps for two seconds to give Linux the chance to execute
+    schedule switching code. Raises an exception if the switch does not work.
+
+    """
+
+    switch_to = str(switch_to_in).strip()
+
+    with open('/proc/litmus/active_plugin', 'w') as active_plugin:
+        subprocess.Popen(["echo", switch_to], stdout=active_plugin)
+
+    # it takes a bit to do the switch, sleep an arbitrary amount of time
+    time.sleep(2)
+
+    with open('/proc/litmus/active_plugin', 'r') as active_plugin:
+        cur_plugin = active_plugin.read().strip()
+
+    if switch_to != cur_plugin:
+        raise Exception("Could not switch to plugin: %s" % switch_to)
+
+
+def is_executable(fname):
+    """Return whether the file passed in is executable"""
+    mode = os.stat(fname)[stat.ST_MODE]
+    return mode & stat.S_IXUSR and mode & stat.S_IRUSR
+
+def is_device(dev):
+    if not os.path.exists(dev):
+        return False
+    mode = os.stat(dev)[stat.ST_MODE]
+    return not (not mode & stat.S_IFCHR)
+
+def release_tasks():
+
+    try:
+        data = subprocess.check_output([conf.BINS['release']])
+    except subprocess.CalledProcessError:
+        raise Exception('Something went wrong in release_ts')
+
+    released = re.findall(r"([0-9]+) real-time", data)[0]
+
+    return int(released)
diff --git a/experiment/proc_entry.py b/experiment/proc_entry.py
new file mode 100644
index 0000000..0b7f9ce
--- /dev/null
+++ b/experiment/proc_entry.py
@@ -0,0 +1,12 @@
+import os
+
+class ProcEntry(object):
+    def __init__(self, proc, data):
+        self.proc = proc
+        self.data = data
+
+    def write_proc(self):
+        if not os.path.exists(self.proc):
+            raise Exception("Invalid proc entry %s" % self.proc)
+        with open(self.proc, 'w') as entry:
+            entry.write(self.data)
diff --git a/experiment/tracer.py b/experiment/tracer.py
new file mode 100644
index 0000000..d7743ad
--- /dev/null
+++ b/experiment/tracer.py
@@ -0,0 +1,118 @@
+import litmus_util
+import os
+from operator import methodcaller
+from executable.ftcat import FTcat,Executable
+from config.config import FILES,BINS
+
+class Tracer(object):
+    def __init__(self, name, output_dir):
+        self.name = name
+        self.output_dir = output_dir
+        self.bins = []
+
+    def start_tracing(self):
+        map(methodcaller("execute"), self.bins)
+
+    def stop_tracing(self):
+        map(methodcaller('terminate'), self.bins)
+        map(methodcaller('wait'), self.bins)
+
+        
+class LinuxTracer(Tracer):
+    EVENT_ROOT = "/sys/kernel/debug/tracing"
+    LITMUS_EVENTS = "%s/events/litmus" % EVENT_ROOT
+
+    def __init__(self, output_dir):
+        super(LinuxTracer, self).__init__("trace-cmd", output_dir)
+        
+        extra_args = ["record", "-e", "sched:sched_switch",
+                      "-e", "litmus:*",
+                      "-o", "%s/%s" % (output_dir, FILES['linux_data'])]
+        stdout = open('%s/trace-cmd-stdout.txt' % self.output_dir, 'w')
+        stderr = open('%s/trace-cmd-stderr.txt' % self.output_dir, 'w')
+        
+        execute = Executable(BINS['trace-cmd'], extra_args, stdout, stderr)
+        self.bins.append(execute)
+        
+    @staticmethod
+    def enabled():
+        return os.path.exists(LinuxTracer.LITMUS_EVENTS)
+
+    def stop_tracing(self):
+        map(methodcaller('interrupt'), self.bins)
+        map(methodcaller('wait'), self.bins)
+
+    
+class LogTracer(Tracer):
+    DEVICE_STR = '/dev/litmus/log'
+
+    def __init__(self, output_dir):
+        super(LogTracer, self).__init__("Logger", output_dir)
+        
+        out_file = open("%s/%s" % (self.output_dir, FILES['log_data']), 'w')
+
+        cat = (Executable("/bin/cat", [LogTracer.DEVICE_STR]))
+        cat.stdout_file = out_file
+
+        self.bins.append(cat)
+
+    @staticmethod
+    def enabled():
+        return litmus_util.is_device(LogTracer.DEVICE_STR)
+    
+
+class SchedTracer(Tracer):
+    EVENTS = range(501, 510) # not including 511
+    DEVICE_STR = '/dev/litmus/sched_trace'
+
+    def __init__(self, output_dir):
+        super(SchedTracer, self).__init__("Sched Trace", output_dir)
+
+        if SchedTracer.enabled():
+            for cpu in range(litmus_util.num_cpus()):
+                # Executable will close the stdout/stderr files
+                stdout_f = open('%s/st-%d.bin' % (self.output_dir, cpu), 'w')
+                stderr_f = open('%s/st-%d-stderr.txt' % (self.output_dir, cpu), 'w')
+                dev = '{0}{1}'.format(SchedTracer.DEVICE_STR, cpu)
+                ftc = FTcat(BINS['ftcat'], stdout_f, stderr_f, dev, SchedTracer.EVENTS, cpu=cpu)
+
+                self.bins.append(ftc)
+
+    @staticmethod
+    def enabled():
+                return litmus_util.is_device("%s%d" % (SchedTracer.DEVICE_STR, 0))
+
+    
+class OverheadTracer(Tracer):
+    DEVICE_STR = '/dev/litmus/ft_trace0'
+    EVENTS = [# 'SCHED_START', 'SCHED_END', 'SCHED2_START', 'SCHED2_END',
+            'RELEASE_START', 'RELEASE_END',
+            'LVLA_RELEASE_START', 'LVLA_RELEASE_END',
+            'LVLA_SCHED_START', 'LVLA_SCHED_END',
+            'LVLB_RELEASE_START', 'LVLB_RELEASE_END',
+            'LVLB_SCHED_START', 'LVLB_SCHED_END',
+            'LVLC_RELEASE_START', 'LVLC_RELEASE_END',
+            'LVLC_SCHED_START', 'LVLC_SCHED_END']
+
+    def __init__(self, output_dir):
+        super(OverheadTracer, self).__init__("Overhead Trace", output_dir)
+
+        stdout_f = open('{0}/{1}'.format(self.output_dir, FILES['ft_data']), 'w')
+        stderr_f = open('{0}/{1}.stderr.txt'.format(self.output_dir, FILES['ft_data']), 'w')
+        ftc = FTcat(BINS['ftcat'], stdout_f, stderr_f,
+                OverheadTracer.DEVICE_STR, OverheadTracer.EVENTS)
+
+        self.bins.append(ftc)
+
+    @staticmethod
+    def enabled():
+                return litmus_util.is_device(OverheadTracer.DEVICE_STR)
+
+
+class PerfTracer(Tracer):
+    def __init__(self, output_dir):
+        super(PerfTracer, self).__init__("CPU perf counters", output_dir)
+
+    @staticmethod
+    def enabled():
+        return False