Use smarter defaults which can handle data from Bjorns scripts.

11 files changed, 92 insertions, 83 deletions

diff --git a/common.py b/common.py
index d080e1a..cba31bf 100644
--- a/common.py
+++ b/common.py
@@ -1,5 +1,6 @@
 import os
 import re
+import stat
 import subprocess
 import sys
 
@@ -134,3 +135,53 @@ def load_params(fname):
         raise IOError("Invalid param file: %s\n%s" % (fname, e))
 
     return params
+
+
+def num_cpus():
+    '''Return the number of CPUs in the system.'''
+
+    lnx_re = re.compile(r'^(processor|online)')
+    cpus = 0
+
+    with open('/proc/cpuinfo', 'r') as f:
+        for line in f:
+            if lnx_re.match(line):
+                cpus += 1
+    return cpus
+
+def ft_freq():
+    umachine = subprocess.check_output(["uname", "-m"])
+
+    if re.match("armv7", umachine):
+        # Arm V7s use a millisecond timer
+        freq = 1000.0
+    elif re.match("x86", umachine):
+        # X86 timer is equal to processor clock
+        reg = re.compile(r'^cpu MHz\s*:\s*(?P<FREQ>\d+)', re.M)
+        with open('/proc/cpuinfo', 'r') as f:
+            data = f.read()
+
+        match = re.search(reg, data)
+        if not match:
+            raise Exception("Cannot parse CPU frequency from x86 CPU!")
+        freq = int(match.group('FREQ'))
+    else:
+        # You're on your own
+        freq = 0
+    return freq
+
+
+def uname_matches(reg):
+    data = subprocess.check_output(["uname", "-r"])
+    return bool( re.match(reg, data) )
+
+def is_executable(fname):
+    '''Return whether the file passed in is executable'''
+    mode = os.stat(fname)[stat.ST_MODE]
+    return mode & stat.S_IXUSR and mode & stat.S_IRUSR
+
+def is_device(dev):
+    if not os.path.exists(dev):
+        return False
+    mode = os.stat(dev)[stat.ST_MODE]
+    return not (not mode & stat.S_IFCHR)
diff --git a/config/config.py b/config/config.py
index 1199278..a6edece 100644
--- a/config/config.py
+++ b/config/config.py
@@ -1,6 +1,6 @@
 from __future__ import print_function
 import itertools
-from common import get_executable
+from common import get_executable,ft_freq
 
 '''Paths to binaries.'''
 BINS = {'rtspin'    : get_executable('rtspin', 'liblitmus'),
@@ -16,6 +16,7 @@ BINS = {'rtspin'    : get_executable('rtspin', 'liblitmus'),
 
 '''Names of output files.'''
 FILES = {'ft_data'    : 'ft.bin',
+         'ft_matches' : r'(ft.*\.bin)|(.*\.ft)',
          'linux_data' : 'trace.dat',
          'sched_data' : 'st-{}.bin',
          'log_data'   : 'trace.slog'}
@@ -34,7 +35,7 @@ DEFAULTS = {'params_file' : 'params.py',
             'sched_file'  : 'sched.py',
             'duration'    : 10,
             'spin'        : 'rtspin',
-            'cycles'      : 2000}
+            'cycles'      : ft_freq() or 2000}
 
 '''Default sched_trace events (this is all of them).'''
 SCHED_EVENTS = range(501, 513)
diff --git a/gen/generator.py b/gen/generator.py
index dbf2a7b..8c3048b 100644
--- a/gen/generator.py
+++ b/gen/generator.py
@@ -1,11 +1,10 @@
 import gen.rv as rv
 import os
-import run.litmus_util as lu
 import shutil as sh
 
 from Cheetah.Template import Template
 from collections import namedtuple
-from common import get_config_option
+from common import get_config_option,num_cpus
 from config.config import DEFAULTS,PARAMS
 from gen.dp import DesignPointGenerator
 from parse.col_map import ColMapBuilder
@@ -69,7 +68,7 @@ class Generator(object):
         if 'cpus' in params:
             cpus = min(map(int, params['cpus']))
         else:
-            cpus = lu.num_cpus()
+            cpus = num_cpus()
         try:
             config = get_config_option("RELEASE_MASTER") and True
         except:
diff --git a/parse/dir_map.py b/parse/dir_map.py
index 601dd3b..a8d2a83 100644
--- a/parse/dir_map.py
+++ b/parse/dir_map.py
@@ -56,6 +56,9 @@ class DirMap(object):
 
         remove_childless2(self.root)
 
+    def is_empty(self):
+        return not len(self.root.children)
+
     def write(self, out_dir):
         def write2(path, node):
             out_path = "/".join(path)
diff --git a/parse/ft.py b/parse/ft.py
index 19453d1..98405f4 100644
--- a/parse/ft.py
+++ b/parse/ft.py
@@ -63,7 +63,7 @@ def extract_ft_data(result, data_dir, work_dir, cycles):
     data_dir = os.path.abspath(data_dir)
     work_dir = os.path.abspath(work_dir)
 
-    freg = conf.FILES['ft_data'] + "$"
+    freg = conf.FILES['ft_matches'] + "$"
     bins = [f for f in os.listdir(data_dir) if re.match(freg, f)]
 
     if not len(bins):
diff --git a/parse/point.py b/parse/point.py
index d577306..f2b266a 100644
--- a/parse/point.py
+++ b/parse/point.py
@@ -128,8 +128,7 @@ class ExpPoint(object):
         self.stats[type] = value
 
     def __str__(self):
-        # return "<ExpPoint-%s>\n%s" % (self.id, dict_str(self.stats))
-        return "<ExpPoint-%s>" % (self.id)
+        return "<ExpPoint-%s>\n%s" % (self.id, dict_str(self.stats))
 
     def get_stats(self):
         return self.stats.keys()
diff --git a/parse_exps.py b/parse_exps.py
index 7dfc9cd..8fcf5c3 100755
--- a/parse_exps.py
+++ b/parse_exps.py
@@ -22,8 +22,6 @@ def parse_args():
     # TODO: convert data-dir to proper option, clean 'dest' options
     parser = OptionParser("usage: %prog [options] [data_dir]...")
 
-    print("default to no params.py")
-
     parser.add_option('-o', '--out', dest='out',
                       help='file or directory for data output', default='parse-data')
     parser.add_option('-c', '--clean', action='store_true', default=False,
@@ -44,15 +42,15 @@ ExpData = namedtuple('ExpData', ['path', 'params', 'work_dir'])
 
 def get_exp_params(data_dir, cm_builder):
     param_file = "%s/%s" % (data_dir, conf.DEFAULTS['params_file'])
-    if not os.path.isfile:
-        raise Exception("No param file '%s' exists!" % param_file)
-
-    params = load_params(param_file)
+    if os.path.isfile(param_file):
+        params = load_params(param_file)
 
-    # Store parameters in cm_builder, which will track which parameters change
-    # across experiments
-    for key, value in params.iteritems():
-        cm_builder.try_add(key, value)
+        # Store parameters in cm_builder, which will track which parameters change
+        # across experiments
+        for key, value in params.iteritems():
+            cm_builder.try_add(key, value)
+    else:
+         params = {}
 
     # Cycles must be present for feather-trace measurement parsing
     if conf.PARAMS['cycles'] not in params:
@@ -164,16 +162,25 @@ def main():
     if opts.force and os.path.exists(opts.out):
         sh.rmtree(opts.out)
 
-    result_table = result_table.reduce()
+    reduced_table = result_table.reduce()
 
     sys.stderr.write("Writing result...\n")
     if opts.write_map:
         # Write summarized results into map
-        result_table.write_map(opts.out)
+        reduced_table.write_map(opts.out)
     else:
         # Write out csv directories for all variable params
-        dir_map = result_table.to_dir_map()
-        dir_map.write(opts.out)
+        dir_map = reduced_table.to_dir_map()
+
+        # No csvs to write, assume user meant to print out data
+        if dir_map.is_empty():
+            sys.stderr.write("Too little data to make csv files.\n")
+            if not opts.verbose:
+                for key, exp in result_table:
+                    for e in exp:
+                        print(e)
+        else:
+            dir_map.write(opts.out)
 
 if __name__ == '__main__':
     main()
diff --git a/run/executable/executable.py b/run/executable/executable.py
index 0a408b7..02e35ae 100644
--- a/run/executable/executable.py
+++ b/run/executable/executable.py
@@ -1,7 +1,7 @@
 import sys
 import subprocess
 import signal
-from ..litmus_util import is_executable
+from common import is_executable
 
 class Executable(object):
     '''Parent object that represents an executable for use in task-sets.'''
diff --git a/run/litmus_util.py b/run/litmus_util.py
index 8a7f87d..b9080c1 100644
--- a/run/litmus_util.py
+++ b/run/litmus_util.py
@@ -1,43 +1,8 @@
 import re
 import time
 import subprocess
-import os
-import stat
 import config.config as conf
 
-def num_cpus():
-    '''Return the number of CPUs in the system.'''
-
-    lnx_re = re.compile(r'^(processor|online)')
-    cpus = 0
-
-    with open('/proc/cpuinfo', 'r') as f:
-        for line in f:
-            if lnx_re.match(line):
-                cpus += 1
-    return cpus
-
-def ft_freq():
-    umachine = subprocess.check_output(["uname", "-m"])
-
-    if re.match("armv7", umachine):
-        # Arm V7s use a millisecond timer
-        freq = 1000.0
-    elif re.match("x86", umachine):
-        # X86 timer is equal to processor clock
-        reg = re.compile(r'^cpu MHz\s*:\s*(?P<FREQ>\d+)', re.M)
-        with open('/proc/cpuinfo', 'r') as f:
-            data = f.read()
-
-        match = re.search(reg, data)
-        if not match:
-            raise Exception("Cannot parse CPU frequency from x86 CPU!")
-        freq = int(match.group('FREQ'))
-    else:
-        # You're on your own
-        freq = 0
-    return freq
-
 def switch_scheduler(switch_to_in):
     '''Switch the scheduler to whatever is passed in.
 
@@ -60,21 +25,6 @@ def switch_scheduler(switch_to_in):
     if switch_to != cur_plugin:
         raise Exception("Could not switch to plugin: %s" % switch_to)
 
-def uname_matches(reg):
-    data = subprocess.check_output(["uname", "-r"])
-    return bool( re.match(reg, data) )
-
-def is_executable(fname):
-    '''Return whether the file passed in is executable'''
-    mode = os.stat(fname)[stat.ST_MODE]
-    return mode & stat.S_IXUSR and mode & stat.S_IRUSR
-
-def is_device(dev):
-    if not os.path.exists(dev):
-        return False
-    mode = os.stat(dev)[stat.ST_MODE]
-    return not (not mode & stat.S_IFCHR)
-
 def waiting_tasks():
     reg = re.compile(r'^ready.*?(?P<READY>\d+)$', re.M)
     with open('/proc/litmus/stats', 'r') as f:
diff --git a/run/tracer.py b/run/tracer.py
index 303aae0..065797c 100644
--- a/run/tracer.py
+++ b/run/tracer.py
@@ -1,7 +1,7 @@
-from . import litmus_util
 import os
 import config.config as conf
 
+from common import is_device,num_cpus
 from operator import methodcaller
 from run.executable.ftcat import FTcat,Executable
 
@@ -58,7 +58,7 @@ class LogTracer(Tracer):
 
     @staticmethod
     def enabled():
-        return litmus_util.is_device(LogTracer.DEVICE_STR)
+        return is_device(LogTracer.DEVICE_STR)
 
     def stop_tracing(self):
         map(methodcaller('interrupt'), self.bins)
@@ -71,7 +71,7 @@ class SchedTracer(Tracer):
         super(SchedTracer, self).__init__("Sched Trace", output_dir)
 
         if SchedTracer.enabled():
-            for cpu in range(litmus_util.num_cpus()):
+            for cpu in range(num_cpus()):
                 # Executable will close the stdout/stderr files
                 stdout_f = open('%s/st-%d.bin' % (self.output_dir, cpu), 'w')
                 stderr_f = open('%s/st-%d-stderr.txt' % (self.output_dir, cpu), 'w')
@@ -83,7 +83,7 @@ class SchedTracer(Tracer):
 
     @staticmethod
     def enabled():
-        return litmus_util.is_device("%s%d" % (SchedTracer.DEVICE_STR, 0))
+        return is_device("%s%d" % (SchedTracer.DEVICE_STR, 0))
 
 class OverheadTracer(Tracer):
     DEVICE_STR = '/dev/litmus/ft_trace0'
@@ -100,7 +100,7 @@ class OverheadTracer(Tracer):
 
     @staticmethod
     def enabled():
-        return litmus_util.is_device(OverheadTracer.DEVICE_STR)
+        return is_device(OverheadTracer.DEVICE_STR)
 
 class PerfTracer(Tracer):
     def __init__(self, output_dir):
diff --git a/run_exps.py b/run_exps.py
index 195d3f8..a05ff93 100755
--- a/run_exps.py
+++ b/run_exps.py
@@ -2,13 +2,12 @@
 from __future__ import print_function
 
 import config.config as conf
-import run.litmus_util as lu
 import os
 import re
 import shutil
 import traceback
 
-from common import load_params,get_executable
+from common import load_params,get_executable,uname_matches,ft_freq
 from optparse import OptionParser
 from run.executable.executable import Executable
 from run.experiment import Experiment,ExperimentDone
@@ -125,7 +124,7 @@ def load_experiment(sched_file, scheduler, duration, param_file, out_dir):
                        (conf.PARAMS['dur'],    duration)])
 
     # Feather-trace clock frequency saved for accurate overhead parsing
-    ft_freq = lu.ft_freq()
+    ft_freq = ft_freq()
     if ft_freq:
         out_params[conf.PARAMS['cycles']] = ft_freq
 
@@ -146,7 +145,7 @@ def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir):
     proc_entries = []
     executables  = []
 
-    if kernel and not lu.uname_matches(kernel):
+    if kernel and not uname_matches(kernel):
         raise InvalidKernel(kernel)
 
     # Parse values for proc entries
@@ -178,7 +177,7 @@ def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir):
         if re.match(".*spin", real_spin):
             real_args = ['-w'] + real_args + [duration]
 
-        if not lu.is_executable(real_spin):
+        if not is_executable(real_spin):
             raise OSError("Cannot run spin %s: %s" % (real_spin, name))
 
         executables += [Executable(real_spin, real_args)]